diff --git a/.github/labeler.yml b/.github/labeler.yml index ee2f4de14..32d610bd5 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -6,10 +6,16 @@ test:uipath-langchain: - changed-files: - any-glob-to-any-file: ['packages/uipath-core/src/**/*.py'] -test:uipath-llamaindex: +test:uipath-integrations: - changed-files: - any-glob-to-any-file: ['packages/uipath/src/**/*.py'] + - changed-files: + - any-glob-to-any-file: ['packages/uipath-platform/src/**/*.py'] + - changed-files: + - any-glob-to-any-file: ['packages/uipath-core/src/**/*.py'] test:uipath-runtime: - changed-files: - any-glob-to-any-file: ['packages/uipath-core/src/**/*.py'] + - changed-files: + - any-glob-to-any-file: ['packages/uipath/pyproject.toml'] diff --git a/.github/scripts/check_dependency_version_bumps.py b/.github/scripts/check_dependency_version_bumps.py new file mode 100644 index 000000000..fef305dea --- /dev/null +++ b/.github/scripts/check_dependency_version_bumps.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Enforce minimum-version bumps between co-changed internal packages. + +The monorepo ships several packages that depend on one another +(``uipath`` -> ``uipath-platform`` -> ``uipath-core``). When a PR changes +the *source* of a dependency package (say ``uipath-core``) **and** the +source of one of its dependents (say ``uipath``), the dependent is almost +certainly relying on the new behaviour. If the dependent does not also +raise the lower bound of its requirement on the dependency, then anyone who +installs the dependent on its own can resolve an older dependency that +predates the new behaviour — a silent runtime break. + +This check fails such a PR. For every pair of co-changed (dependency, +dependent) packages it requires the dependent's lower-bound constraint on +the dependency (the ``>=`` part of e.g. ``uipath-core>=0.5.8, <0.6.0``) to +be at least the dependency's new version declared in this PR. + +The internal dependency graph is discovered from the pyproject files, so no +hard-coded list needs maintaining as packages are added. +""" + +import re +import sys +from pathlib import Path +from typing import TypedDict + +from check_version_uniqueness import get_changed_packages + +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib # type: ignore[no-redef] + +PACKAGES_DIR = Path("packages") + + +class PackageInfo(TypedDict): + """Resolved metadata for a single monorepo package.""" + + dir: str + name: str + version: str + dependencies: list[str] + + +def normalize_name(name: str) -> str: + """Normalize a PyPI project name (PEP 503): case-insensitive, -/_/. + treated as equivalent.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def version_key(version: str) -> tuple[int, ...]: + """Numeric sort key so ``0.5.17`` > ``0.5.8`` (``0.5.18rc1`` -> ``(0, 5, 18)``).""" + parts: list[int] = [] + for component in version.split("."): + digits = "" + for ch in component: + if ch.isdigit(): + digits += ch + else: + break + parts.append(int(digits) if digits else 0) + return tuple(parts) + + +def parse_requirement(requirement: str) -> tuple[str | None, str | None]: + """Extract (normalized name, lower-bound version) from a requirement string. + + Returns the lower bound found in a ``>=`` clause, or ``None`` if there is + no ``>=`` constraint. The name is ``None`` if the string is unparseable. + """ + name_match = re.match(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*)", requirement) + if not name_match: + return None, None + name = normalize_name(name_match.group(1)) + + lower: str | None = None + lower_match = re.search(r">=\s*([0-9][0-9A-Za-z._-]*)", requirement) + if lower_match: + lower = lower_match.group(1) + return name, lower + + +def load_package(package_dir: str) -> PackageInfo | None: + """Read a package's name, version and dependency list from pyproject.toml.""" + pyproject = PACKAGES_DIR / package_dir / "pyproject.toml" + if not pyproject.exists(): + return None + with open(pyproject, "rb") as f: + data = tomllib.load(f) + project = data.get("project", {}) + name = project.get("name") + version = project.get("version") + if not name or not version: + return None + return PackageInfo( + dir=package_dir, + name=name, + version=version, + dependencies=list(project.get("dependencies", [])), + ) + + +def get_all_packages() -> dict[str, PackageInfo]: + """Map package directory name -> package info for every package.""" + packages: dict[str, PackageInfo] = {} + if not PACKAGES_DIR.is_dir(): + return packages + for item in sorted(PACKAGES_DIR.iterdir()): + if item.is_dir() and (item / "pyproject.toml").exists(): + info = load_package(item.name) + if info: + packages[item.name] = info + return packages + + +def check(packages: dict[str, PackageInfo], changed: set[str]) -> list[str]: + """Return a list of violation messages (empty when the PR is compliant).""" + name_to_dir: dict[str, str] = { + normalize_name(info["name"]): pkg_dir for pkg_dir, info in packages.items() + } + + violations: list[str] = [] + for dependent_dir in sorted(changed): + dependent = packages.get(dependent_dir) + if not dependent: + continue + + for requirement in dependent["dependencies"]: + dep_name, lower = parse_requirement(requirement) + if dep_name is None: + continue + + dep_dir = name_to_dir.get(dep_name) + # Only internal packages that *also* changed in this PR are in scope. + if dep_dir is None or dep_dir == dependent_dir or dep_dir not in changed: + continue + + dep_version = packages[dep_dir]["version"] + dep_display = packages[dep_dir]["name"] + + if lower is None: + violations.append( + f"{dependent['name']} requires '{requirement}' but has no '>=' lower bound on " + f"{dep_display}; pin it to >={dep_version} (both packages changed in this PR)." + ) + elif version_key(lower) < version_key(dep_version): + violations.append( + f"{dependent['name']} pins {dep_display}>={lower}, but {dep_display} was bumped to " + f"{dep_version} in this PR. Raise the minimum to >={dep_version}." + ) + else: + print(f"OK: {dependent['name']} requires {dep_display}>={lower} (>= new {dep_version})") + + return violations + + +def main() -> int: + packages = get_all_packages() + if not packages: + print("No packages found.") + return 0 + + changed = set(get_changed_packages()) + if not changed: + print("No source changes to internal packages detected.") + return 0 + + print(f"Changed packages: {', '.join(sorted(changed))}") + + violations = check(packages, changed) + if violations: + print("\nDependency version bump check FAILED:\n", file=sys.stderr) + for v in violations: + print(f" - {v}", file=sys.stderr) + print( + "\nWhen you change an internal package and a dependent of it in the same PR, " + "the dependent must require the dependency's new version so a standalone install " + "cannot resolve an older, incompatible release.", + file=sys.stderr, + ) + return 1 + + print("\nAll co-changed internal dependencies have an up-to-date minimum version.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/.github/scripts/test_check_dependency_version_bumps.py b/.github/scripts/test_check_dependency_version_bumps.py new file mode 100644 index 000000000..e4135e89f --- /dev/null +++ b/.github/scripts/test_check_dependency_version_bumps.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Tests for check_dependency_version_bumps.py.""" + +from unittest import mock + +from check_dependency_version_bumps import ( + PackageInfo, + check, + normalize_name, + parse_requirement, + version_key, +) + + +def pkg(name: str, version: str, dependencies: list[str] | None = None) -> PackageInfo: + return PackageInfo( + dir=name, + name=name, + version=version, + dependencies=dependencies or [], + ) + + +class TestVersionKey: + def test_numeric_components(self): + assert version_key("0.5.18") == (0, 5, 18) + + def test_compares_numerically_not_lexically(self): + # The trap a string compare would fall into: "0.5.8" > "0.5.17". + assert version_key("0.5.17") > version_key("0.5.8") + + def test_strips_prerelease_suffix(self): + assert version_key("0.5.18rc1") == (0, 5, 18) + + +class TestNormalizeName: + def test_case_and_separators_equivalent(self): + assert normalize_name("UiPath_Core") == normalize_name("uipath-core") + assert normalize_name("uipath.core") == "uipath-core" + + +class TestParseRequirement: + def test_extracts_name_and_lower_bound(self): + assert parse_requirement("uipath-core>=0.5.8, <0.6.0") == ("uipath-core", "0.5.8") + + def test_no_lower_bound(self): + assert parse_requirement("click") == ("click", None) + assert parse_requirement("httpx<1.0") == ("httpx", None) + + def test_whitespace_after_operator(self): + assert parse_requirement("uipath-core >= 0.5.8") == ("uipath-core", "0.5.8") + + +class TestCheck: + def _packages(self) -> dict[str, PackageInfo]: + return { + "uipath-core": pkg("uipath-core", "0.5.18"), + "uipath-platform": pkg( + "uipath-platform", "0.1.60", ["uipath-core>=0.5.8, <0.6.0"] + ), + "uipath": pkg( + "uipath", + "2.10.74", + [ + "uipath-core>=0.5.8, <0.6.0", + "uipath-platform>=0.1.59, <0.2.0", + "click>=8.3.1", + ], + ), + } + + def test_passes_when_only_dependency_changed(self): + # uipath-core changed alone -> dependents not touched, nothing to enforce. + assert check(self._packages(), {"uipath-core"}) == [] + + def test_passes_when_only_dependent_changed(self): + assert check(self._packages(), {"uipath"}) == [] + + def test_fails_when_co_changed_without_min_bump(self): + # uipath-core bumped to 0.5.18 but uipath still pins >=0.5.8. + violations = check(self._packages(), {"uipath-core", "uipath"}) + assert len(violations) == 1 + assert "uipath" in violations[0] + assert "0.5.18" in violations[0] + + def test_passes_when_min_raised_to_new_version(self): + packages = self._packages() + packages["uipath"]["dependencies"] = [ + "uipath-core>=0.5.18, <0.6.0", + "click>=8.3.1", + ] + assert check(packages, {"uipath-core", "uipath"}) == [] + + def test_passes_when_min_already_above_new_version(self): + packages = self._packages() + packages["uipath"]["dependencies"] = ["uipath-core>=0.6.0, <0.7.0"] + assert check(packages, {"uipath-core", "uipath"}) == [] + + def test_fails_when_no_lower_bound_on_co_changed_dep(self): + packages = self._packages() + packages["uipath"]["dependencies"] = ["uipath-core"] + violations = check(packages, {"uipath-core", "uipath"}) + assert len(violations) == 1 + assert "no '>=' lower bound" in violations[0] + + def test_external_dependencies_are_ignored(self): + # click is not an internal package, so it is never in scope. + assert check(self._packages(), {"uipath"}) == [] + + def test_transitive_chain_each_edge_enforced(self): + # All three changed: uipath must bump core AND platform; platform must bump core. + packages = self._packages() + violations = check(packages, {"uipath-core", "uipath-platform", "uipath"}) + # uipath->core (stale), uipath->platform (0.1.59 < 0.1.60), platform->core (stale) + assert len(violations) == 3 + + def test_no_self_reference(self): + packages = {"uipath": pkg("uipath", "2.0.0", ["uipath>=1.0.0"])} + assert check(packages, {"uipath"}) == [] + + +class TestMain: + def _run(self, packages: dict[str, PackageInfo], changed: list[str]) -> int: + from check_dependency_version_bumps import main + + with ( + mock.patch("check_dependency_version_bumps.get_all_packages", return_value=packages), + mock.patch("check_dependency_version_bumps.get_changed_packages", return_value=changed), + ): + return main() + + def test_returns_zero_when_compliant(self): + packages = { + "uipath-core": pkg("uipath-core", "0.5.18"), + "uipath": pkg("uipath", "2.0.0", ["uipath-core>=0.5.18, <0.6.0"]), + } + assert self._run(packages, ["uipath-core", "uipath"]) == 0 + + def test_returns_one_on_violation(self): + packages = { + "uipath-core": pkg("uipath-core", "0.5.18"), + "uipath": pkg("uipath", "2.0.0", ["uipath-core>=0.5.8, <0.6.0"]), + } + assert self._run(packages, ["uipath-core", "uipath"]) == 1 + + def test_returns_zero_when_no_changes(self): + assert self._run({"uipath-core": pkg("uipath-core", "0.5.18")}, []) == 0 \ No newline at end of file diff --git a/.github/scripts/test_check_version_uniqueness.py b/.github/scripts/test_check_version_uniqueness.py index 94a2b3cc0..af4298af1 100644 --- a/.github/scripts/test_check_version_uniqueness.py +++ b/.github/scripts/test_check_version_uniqueness.py @@ -5,7 +5,6 @@ import urllib.error from unittest import mock -import pytest from check_version_uniqueness import ( get_package_info, diff --git a/.github/scripts/write_uv_overrides.py b/.github/scripts/write_uv_overrides.py new file mode 100644 index 000000000..557f9d26e --- /dev/null +++ b/.github/scripts/write_uv_overrides.py @@ -0,0 +1,50 @@ +"""Write a uv override file forcing the locally built uipath wheels. + +Cross-test workflows build uipath wheels from the PR and run them against +downstream repos (uipath-langchain-python, uipath-integrations-python, +uipath-runtime-python). Those downstreams cap the uipath* version (e.g. +``uipath<2.11.0``), so a backward-compatible minor bump would fail resolution +purely on the cap. uv ``override-dependencies`` ignore the declared version +specifier, so pointing them at the local wheels lets the cross-test exercise the +real new code regardless of the cap. + +The script is layout-agnostic: it overrides whatever ``uipath*`` wheels exist +under ``$GITHUB_WORKSPACE/wheels`` (recursively), so it works for the +three-wheel layout (``wheels//dist/*.whl``) and the single-wheel runtime +layout (``wheels/*.whl``) alike. + +The resulting override file path is appended to ``GITHUB_ENV`` as ``UV_OVERRIDE`` +so every subsequent ``uv`` invocation in the job honors it. +""" + +import glob +import os +import pathlib + + +def main() -> None: + wheels = pathlib.Path(os.environ.get("GITHUB_WORKSPACE", ".")).resolve() / "wheels" + + lines = [] + for whl in sorted(glob.glob(str(wheels / "**" / "*.whl"), recursive=True)): + # Wheel filename is ``{distribution}-{version}-...whl`` where the + # distribution escapes hyphens to underscores (uipath_core -> uipath-core). + dist = pathlib.Path(whl).name.split("-", 1)[0].replace("_", "-") + if not dist.startswith("uipath"): + continue + lines.append(f"{dist} @ {pathlib.Path(whl).resolve().as_uri()}") + + if not lines: + raise SystemExit(f"no uipath wheels found under {wheels}") + + out = wheels / "overrides.txt" + out.write_text("\n".join(lines) + "\n") + + with open(os.environ["GITHUB_ENV"], "a") as fh: + fh.write(f"UV_OVERRIDE={out}\n") + + print("\n".join(lines)) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/auto-label.yml b/.github/workflows/auto-label.yml index a5f67bcc7..4bfccae4c 100644 --- a/.github/workflows/auto-label.yml +++ b/.github/workflows/auto-label.yml @@ -10,10 +10,10 @@ permissions: jobs: label: - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest steps: - name: Label PR - uses: actions/labeler@v5 + uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9 # v5.0.0 with: configuration-path: '.github/labeler.yml' sync-labels: true diff --git a/.github/workflows/build-package.yml b/.github/workflows/build-package.yml index 54cd05352..faaf7df3f 100644 --- a/.github/workflows/build-package.yml +++ b/.github/workflows/build-package.yml @@ -23,21 +23,21 @@ permissions: jobs: build: name: Build ${{ inputs.package }} - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest defaults: run: working-directory: packages/${{ inputs.package }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Setup uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 with: enable-cache: true - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version-file: "packages/${{ inputs.package }}/.python-version" @@ -72,7 +72,7 @@ jobs: run: uv build --no-sources --package ${{ inputs.package }} - name: Upload artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: release-dists-${{ inputs.package }} path: packages/${{ inputs.package }}/dist/ diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 82aa4ccb2..434e9d57d 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -18,16 +18,16 @@ permissions: jobs: detect-publishable-packages: - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest outputs: packages: ${{ steps.detect.outputs.packages }} count: ${{ steps.detect.outputs.count }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.11' @@ -48,20 +48,20 @@ jobs: name: Publish uipath-core needs: [detect-publishable-packages, build-uipath-core] if: contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath-core') - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest environment: pypi permissions: contents: read id-token: write steps: - name: Retrieve release distributions - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: release-dists-uipath-core path: dist/ - name: Publish package distributions to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 with: verbose: true skip-existing: true @@ -73,15 +73,15 @@ jobs: always() && (contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath-platform') || contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath')) - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest steps: - name: Checkout if: contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath-core') - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Setup Python if: contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath-core') - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.11' @@ -110,20 +110,20 @@ jobs: if: | always() && contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath-platform') - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest environment: pypi permissions: contents: read id-token: write steps: - name: Retrieve release distributions - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: release-dists-uipath-platform path: dist/ - name: Publish package distributions to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 with: verbose: true skip-existing: true @@ -134,15 +134,15 @@ jobs: if: | always() && contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath') - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest steps: - name: Checkout if: contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath-platform') - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Setup Python if: contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath-platform') - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.11' @@ -173,20 +173,20 @@ jobs: if: | always() && contains(fromJson(needs.detect-publishable-packages.outputs.packages), 'uipath') - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest environment: pypi permissions: contents: read id-token: write steps: - name: Retrieve release distributions - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: release-dists-uipath path: dist/ - name: Publish package distributions to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 with: verbose: true skip-existing: true diff --git a/.github/workflows/check-dependency-bumps.yml b/.github/workflows/check-dependency-bumps.yml new file mode 100644 index 000000000..8cf3de004 --- /dev/null +++ b/.github/workflows/check-dependency-bumps.yml @@ -0,0 +1,28 @@ +name: Check Dependency Version Bumps + +on: + workflow_call: + +permissions: + contents: read + +jobs: + check-dependency-bumps: + runs-on: uipath-ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.11' + + - name: Enforce min-version bumps for co-changed internal packages + env: + GITHUB_EVENT_NAME: ${{ github.event_name }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: python .github/scripts/check_dependency_version_bumps.py \ No newline at end of file diff --git a/.github/workflows/check-version-availability.yml b/.github/workflows/check-version-availability.yml index 02ace93c1..8c0897375 100644 --- a/.github/workflows/check-version-availability.yml +++ b/.github/workflows/check-version-availability.yml @@ -8,15 +8,15 @@ permissions: jobs: check-version-availability: - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.11' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e05e9c73..1dd0471f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,9 @@ on: pull_request: branches: - main + push: + branches: + - main permissions: contents: read @@ -18,6 +21,13 @@ jobs: test: uses: ./.github/workflows/test-packages.yml + secrets: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} check-versions: + if: github.event_name == 'pull_request' uses: ./.github/workflows/check-version-availability.yml + + check-dependency-bumps: + if: github.event_name == 'pull_request' + uses: ./.github/workflows/check-dependency-bumps.yml diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml index 8562e455d..d88c4910f 100644 --- a/.github/workflows/commitlint.yml +++ b/.github/workflows/commitlint.yml @@ -6,18 +6,18 @@ on: jobs: commitlint: name: Commit Lint - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest permissions: contents: read steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 - name: Setup Node - uses: actions/setup-node@v3 + uses: actions/setup-node@3235b876344d2a9aa001b8d1453c930bba69e610 # v3.9.1 with: node-version: 22 diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 69311181f..4e9ba0e71 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -11,18 +11,18 @@ permissions: jobs: detect-changed-packages: - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest outputs: packages: ${{ steps.detect.outputs.packages }} count: ${{ steps.detect.outputs.count }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.11' @@ -37,13 +37,13 @@ jobs: discover-testcases: needs: [detect-changed-packages] if: needs.detect-changed-packages.outputs.count > 0 - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest outputs: testcases: ${{ steps.discover.outputs.testcases }} count: ${{ steps.discover.outputs.count }} steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Discover testcases id: discover @@ -75,7 +75,7 @@ jobs: integration-tests: needs: [discover-testcases] if: needs.discover-testcases.outputs.count > 0 - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest container: image: ghcr.io/astral-sh/uv:python3.12-bookworm env: @@ -91,7 +91,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Install dependencies working-directory: packages/${{ matrix.testcase.package }} @@ -125,7 +125,7 @@ jobs: summarize-results: needs: [detect-changed-packages, discover-testcases, integration-tests] - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest if: always() steps: - name: Check integration tests status diff --git a/.github/workflows/lint-packages.yml b/.github/workflows/lint-packages.yml index d530e25a9..c005b8ba9 100644 --- a/.github/workflows/lint-packages.yml +++ b/.github/workflows/lint-packages.yml @@ -8,18 +8,18 @@ permissions: jobs: detect-changed-packages: - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest outputs: packages: ${{ steps.detect.outputs.packages }} count: ${{ steps.detect.outputs.count }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.11' @@ -34,7 +34,7 @@ jobs: lint-uipath-core: name: Lint uipath-core needs: detect-changed-packages - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest steps: - name: Check if package changed id: check @@ -51,17 +51,17 @@ jobs: - name: Checkout if: steps.check.outputs.skip != 'true' - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Setup uv if: steps.check.outputs.skip != 'true' - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 with: enable-cache: true - name: Setup Python if: steps.check.outputs.skip != 'true' - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version-file: "packages/uipath-core/.python-version" @@ -88,7 +88,7 @@ jobs: lint-uipath-platform: name: Lint uipath-platform needs: detect-changed-packages - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest steps: - name: Check if package changed id: check @@ -105,17 +105,17 @@ jobs: - name: Checkout if: steps.check.outputs.skip != 'true' - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Setup uv if: steps.check.outputs.skip != 'true' - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 with: enable-cache: true - name: Setup Python if: steps.check.outputs.skip != 'true' - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version-file: "packages/uipath-platform/.python-version" @@ -142,7 +142,7 @@ jobs: lint-uipath: name: Lint uipath needs: detect-changed-packages - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest steps: - name: Check if package changed id: check @@ -159,17 +159,17 @@ jobs: - name: Checkout if: steps.check.outputs.skip != 'true' - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Setup uv if: steps.check.outputs.skip != 'true' - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 with: enable-cache: true - name: Setup Python if: steps.check.outputs.skip != 'true' - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version-file: "packages/uipath/.python-version" @@ -201,7 +201,7 @@ jobs: lint-gate: name: Lint needs: [lint-uipath-core, lint-uipath-platform, lint-uipath] - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest if: always() steps: - name: Check lint results diff --git a/.github/workflows/publish-dev.yml b/.github/workflows/publish-dev.yml index 3c438f75c..a24503feb 100644 --- a/.github/workflows/publish-dev.yml +++ b/.github/workflows/publish-dev.yml @@ -15,18 +15,18 @@ permissions: jobs: detect-changed-packages: if: contains(github.event.pull_request.labels.*.name, 'build:dev') - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest outputs: packages: ${{ steps.detect.outputs.packages }} count: ${{ steps.detect.outputs.count }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.11' @@ -42,7 +42,7 @@ jobs: name: Publish Dev Build - ${{ matrix.package }} needs: detect-changed-packages if: contains(github.event.pull_request.labels.*.name, 'build:dev') && needs.detect-changed-packages.outputs.count > 0 - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest defaults: run: working-directory: packages/${{ matrix.package }} @@ -53,15 +53,15 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Setup uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 with: enable-cache: true - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version-file: "packages/${{ matrix.package }}/.python-version" @@ -109,17 +109,53 @@ jobs: Write-Output "Package $PROJECT_NAME version set to $DEV_VERSION" + # Shared dev suffix for every package built in this run (same PR + run number) + $DEV_SUFFIX = "dev1$PADDED_PR$PADDED_RUN" + + # Intra-repo dependencies per package. A dev build of these is < the package's + # base version (PEP 440 pre-release), so it falls outside the published ">=base" + # constraint and must be forced in via [tool.uv] override-dependencies. + $internalDepsMap = @{ + "uipath" = @("uipath-platform", "uipath-core") + "uipath-platform" = @("uipath-core") + "uipath-core" = @() + } + + # Packages also published in this run (their dev builds exist on testpypi) + $changedPackages = '${{ needs.detect-changed-packages.outputs.packages }}' | ConvertFrom-Json + + $overrideDeps = @() + foreach ($dep in $internalDepsMap[$PROJECT_NAME]) { + if ($changedPackages -contains $dep) { + $depPyproj = Get-Content "../$dep/pyproject.toml" -Raw + $depBaseVersion = ($depPyproj | Select-String -Pattern '(?m)^\[(project|tool\.poetry)\][^\[]*?version\s*=\s*"([^"]*)"' -AllMatches).Matches[0].Groups[2].Value + $overrideDeps += [PSCustomObject]@{ Name = $dep; Version = "$depBaseVersion.$DEV_SUFFIX" } + } + } + + # [tool.uv.sources]: the package itself plus every overridden dep point at testpypi + $sourcesLines = @("$PROJECT_NAME = { index = `"testpypi`" }") + foreach ($d in $overrideDeps) { $sourcesLines += "$($d.Name) = { index = `"testpypi`" }" } + $sourcesBlock = $sourcesLines -join "`n" + + # Optional [tool.uv] override block (omitted when no intra-repo dep was published) + $overrideBlock = "" + if ($overrideDeps.Count -gt 0) { + $overrideItems = ($overrideDeps | ForEach-Object { "`"$($_.Name)==$($_.Version)`"" }) -join ", " + $overrideBlock = "`n[tool.uv]`noverride-dependencies = [$overrideItems]`n" + } + $dependencyMessage = @" ### $PROJECT_NAME ``````toml [project] dependencies = [ - # Exact version: + # Exact version (copy-paste ready): "$PROJECT_NAME==$DEV_VERSION", - # Any version from PR - "$PROJECT_NAME>=$MIN_VERSION,<$MAX_VERSION" + # Any version from this PR (uncomment to use a range instead): + # "$PROJECT_NAME>=$MIN_VERSION,<$MAX_VERSION", ] [[tool.uv.index]] @@ -129,8 +165,8 @@ jobs: explicit = true [tool.uv.sources] - $PROJECT_NAME = { index = "testpypi" } - `````` + $sourcesBlock + $overrideBlock`````` "@ # Get the owner and repo from the GitHub repository diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index 79eec3cc1..7bfd790a2 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -9,27 +9,31 @@ on: - "packages/uipath/docs/**" - "packages/uipath/mkdocs.yml" - "packages/uipath/pyproject.toml" + - "packages/uipath-platform/src/**" + - "packages/uipath-platform/pyproject.toml" + - "packages/uipath-core/src/**" + - "packages/uipath-core/pyproject.toml" repository_dispatch: types: [publish-docs] jobs: publish-docs: - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest if: ${{ github.repository == 'UiPath/uipath-python' }} permissions: contents: write steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Setup uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 with: enable-cache: true - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version-file: "packages/uipath/.python-version" diff --git a/.github/workflows/test-cd-scripts.yml b/.github/workflows/test-cd-scripts.yml index a2f1bcdf1..0ac5e5f86 100644 --- a/.github/workflows/test-cd-scripts.yml +++ b/.github/workflows/test-cd-scripts.yml @@ -9,21 +9,24 @@ on: - '.github/scripts/test_detect_publishable_packages.py' - '.github/scripts/check_version_uniqueness.py' - '.github/scripts/test_check_version_uniqueness.py' + - '.github/scripts/check_dependency_version_bumps.py' + - '.github/scripts/test_check_dependency_version_bumps.py' - '.github/workflows/cd.yml' - '.github/workflows/check-version-availability.yml' + - '.github/workflows/check-dependency-bumps.yml' permissions: contents: read jobs: test-cd-scripts: - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.11' @@ -32,4 +35,4 @@ jobs: - name: Run tests working-directory: .github/scripts - run: python -m pytest test_detect_publishable_packages.py test_check_version_uniqueness.py -v + run: python -m pytest test_detect_publishable_packages.py test_check_version_uniqueness.py test_check_dependency_version_bumps.py -v diff --git a/.github/workflows/test-packages.yml b/.github/workflows/test-packages.yml index 58e37a42a..b6f5f6428 100644 --- a/.github/workflows/test-packages.yml +++ b/.github/workflows/test-packages.yml @@ -2,25 +2,28 @@ name: Test Packages on: workflow_call: + secrets: + SONAR_TOKEN: + required: false permissions: contents: read jobs: detect-changed-packages: - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest outputs: packages: ${{ steps.detect.outputs.packages }} count: ${{ steps.detect.outputs.count }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.11" @@ -40,7 +43,7 @@ jobs: fail-fast: false matrix: python-version: ["3.11", "3.12", "3.13"] - os: [ubuntu-latest, windows-latest] + os: [uipath-ubuntu-latest, uipath-windows-latest] steps: - name: Check if package changed id: check @@ -59,15 +62,15 @@ jobs: - name: Checkout if: steps.check.outputs.skip != 'true' - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Setup uv if: steps.check.outputs.skip != 'true' - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 - name: Setup Python if: steps.check.outputs.skip != 'true' - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} @@ -77,10 +80,31 @@ jobs: run: uv sync --all-extras --python ${{ matrix.python-version }} - name: Run tests - if: steps.check.outputs.skip != 'true' + if: steps.check.outputs.skip != 'true' && !(matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13') working-directory: packages/uipath-core run: uv run pytest + - name: Run tests with coverage + if: steps.check.outputs.skip != 'true' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13' + working-directory: packages/uipath-core + run: uv run pytest --cov-report=xml --cov-report=html --tb=short + + - name: Upload coverage HTML report + if: steps.check.outputs.skip != 'true' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13' && always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: coverage-html-uipath-core + path: packages/uipath-core/htmlcov/ + retention-days: 30 + + - name: Upload coverage XML report + if: steps.check.outputs.skip != 'true' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13' && always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: coverage-xml-uipath-core + path: packages/uipath-core/coverage.xml + retention-days: 30 + test-uipath-platform: name: Test (uipath-platform, ${{ matrix.python-version }}, ${{ matrix.os }}) needs: detect-changed-packages @@ -89,7 +113,7 @@ jobs: fail-fast: false matrix: python-version: ["3.11", "3.12", "3.13"] - os: [ubuntu-latest, windows-latest] + os: [uipath-ubuntu-latest, uipath-windows-latest] steps: - name: Check if package changed id: check @@ -108,15 +132,15 @@ jobs: - name: Checkout if: steps.check.outputs.skip != 'true' - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Setup uv if: steps.check.outputs.skip != 'true' - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 - name: Setup Python if: steps.check.outputs.skip != 'true' - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} @@ -126,10 +150,80 @@ jobs: run: uv sync --all-extras --python ${{ matrix.python-version }} - name: Run tests - if: steps.check.outputs.skip != 'true' + if: steps.check.outputs.skip != 'true' && !(matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13') working-directory: packages/uipath-platform run: uv run pytest + - name: Run tests with coverage + if: steps.check.outputs.skip != 'true' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13' + working-directory: packages/uipath-platform + run: uv run pytest --cov-report=xml --cov-report=html --tb=short + + - name: Upload coverage HTML report + if: steps.check.outputs.skip != 'true' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13' && always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: coverage-html-uipath-platform + path: packages/uipath-platform/htmlcov/ + retention-days: 30 + + - name: Upload coverage XML report + if: steps.check.outputs.skip != 'true' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13' && always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: coverage-xml-uipath-platform + path: packages/uipath-platform/coverage.xml + retention-days: 30 + + e2e-uipath-platform: + name: E2E (uipath-platform, memory) + needs: detect-changed-packages + runs-on: uipath-ubuntu-latest + steps: + - name: Check if package changed + id: check + shell: bash + run: | + if echo '${{ needs.detect-changed-packages.outputs.packages }}' | jq -e 'index("uipath-platform")' > /dev/null; then + echo "skip=false" >> $GITHUB_OUTPUT + else + echo "skip=true" >> $GITHUB_OUTPUT + fi + + - name: Skip + if: steps.check.outputs.skip == 'true' + shell: bash + run: echo "Skipping - no changes to uipath-platform" + + - name: Checkout + if: steps.check.outputs.skip != 'true' + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + + - name: Setup uv + if: steps.check.outputs.skip != 'true' + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 + + - name: Setup Python + if: steps.check.outputs.skip != 'true' + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.11" + + - name: Install dependencies + if: steps.check.outputs.skip != 'true' + working-directory: packages/uipath-platform + run: uv sync --all-extras --python 3.11 + + - name: Run E2E memory tests + if: steps.check.outputs.skip != 'true' + working-directory: packages/uipath-platform + env: + UIPATH_URL: ${{ secrets.ALPHA_BASE_URL }} + UIPATH_CLIENT_ID: ${{ secrets.ALPHA_TEST_CLIENT_ID }} + UIPATH_CLIENT_SECRET: ${{ secrets.ALPHA_TEST_CLIENT_SECRET }} + UIPATH_FOLDER_KEY: ${{ secrets.UIPATH_MEMORY_FOLDER }} + run: uv run pytest tests/services/test_memory_service_e2e.py -m e2e -v --no-cov + test-uipath: name: Test (uipath, ${{ matrix.python-version }}, ${{ matrix.os }}) needs: detect-changed-packages @@ -139,7 +233,7 @@ jobs: fail-fast: false matrix: python-version: ["3.11", "3.12", "3.13"] - os: [ubuntu-latest, windows-latest] + os: [uipath-ubuntu-latest, uipath-windows-latest] steps: - name: Check if package changed id: check @@ -158,15 +252,15 @@ jobs: - name: Checkout if: steps.check.outputs.skip != 'true' - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Setup uv if: steps.check.outputs.skip != 'true' - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 - name: Setup Python if: steps.check.outputs.skip != 'true' - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} @@ -176,16 +270,82 @@ jobs: run: uv sync --all-extras --python ${{ matrix.python-version }} - name: Run tests - if: steps.check.outputs.skip != 'true' + if: steps.check.outputs.skip != 'true' && !(matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13') working-directory: packages/uipath run: uv run pytest + - name: Run tests with coverage + if: steps.check.outputs.skip != 'true' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13' + working-directory: packages/uipath + run: uv run pytest --cov-report=xml --cov-report=html --tb=short + + - name: Upload coverage HTML report + if: steps.check.outputs.skip != 'true' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13' && always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: coverage-html-uipath + path: packages/uipath/htmlcov/ + retention-days: 30 + + - name: Upload coverage XML report + if: steps.check.outputs.skip != 'true' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13' && always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: coverage-xml-uipath + path: packages/uipath/coverage.xml + retention-days: 30 + continue-on-error: true + sonarcloud: + name: SonarCloud + needs: [test-uipath-core, test-uipath-platform, test-uipath] + runs-on: uipath-ubuntu-latest + if: always() && needs.test-uipath-core.result != 'failure' && needs.test-uipath-platform.result != 'failure' && needs.test-uipath.result != 'failure' + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + fetch-depth: 0 + + - name: Download uipath-core coverage + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + continue-on-error: true + with: + name: coverage-xml-uipath-core + path: packages/uipath-core + + - name: Download uipath-platform coverage + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + continue-on-error: true + with: + name: coverage-xml-uipath-platform + path: packages/uipath-platform + + - name: Download uipath coverage + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + continue-on-error: true + with: + name: coverage-xml-uipath + path: packages/uipath + + - name: Rewrite coverage XML to repo-relative paths + run: | + sed -i 's|src|packages/uipath-core/src|g' packages/uipath-core/coverage.xml || true + sed -i 's|src|packages/uipath-platform/src|g' packages/uipath-platform/coverage.xml || true + sed -i 's|src|packages/uipath/src|g' packages/uipath/coverage.xml || true + + - name: SonarCloud Scan + uses: SonarSource/sonarqube-scan-action@2f77a1ec69fb1d595b06f35ab27e97605bdef703 # v5 + env: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + test-gate: name: Test - needs: [test-uipath-core, test-uipath-platform, test-uipath] - runs-on: ubuntu-latest + needs: [test-uipath-core, test-uipath-platform, test-uipath, e2e-uipath-platform] + runs-on: uipath-ubuntu-latest if: always() steps: - name: Check test results @@ -196,4 +356,8 @@ jobs: echo "Tests failed" exit 1 fi + # E2E tests are informational — log but don't block + if [[ "${{ needs.e2e-uipath-platform.result }}" == "failure" ]]; then + echo "⚠️ E2E memory tests failed (non-blocking)" + fi echo "All tests passed" diff --git a/.github/workflows/test-uipath-integrations.yml b/.github/workflows/test-uipath-integrations.yml new file mode 100644 index 000000000..6bf5ceef0 --- /dev/null +++ b/.github/workflows/test-uipath-integrations.yml @@ -0,0 +1,292 @@ +name: uipath - Test Integrations + +on: + pull_request: + types: [ opened, synchronize, reopened, labeled ] + +jobs: + build-wheels: + runs-on: uipath-ubuntu-latest + permissions: + contents: read + if: contains(github.event.pull_request.labels.*.name, 'test:uipath-integrations') + steps: + - name: Checkout uipath-python + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + + - name: Setup uv + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 + + - name: Setup Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Build uipath-core package + working-directory: packages/uipath-core + run: uv build + + - name: Build uipath-platform package + working-directory: packages/uipath-platform + run: uv build + + - name: Build uipath package + working-directory: packages/uipath + run: uv build + + - name: Upload wheels + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: uipath-wheels + path: packages/*/dist/*.whl + + discover-packages: + needs: [build-wheels] + runs-on: uipath-ubuntu-latest + permissions: + contents: read + outputs: + packages: ${{ steps.discover.outputs.packages }} + steps: + - name: Checkout uipath-integrations-python + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + repository: 'UiPath/uipath-integrations-python' + path: 'uipath-integrations-python' + + - name: Discover packages + id: discover + working-directory: uipath-integrations-python + run: | + # Find every package directory under packages/ that has a pyproject.toml + package_dirs=$(find packages -maxdepth 2 -name pyproject.toml -printf '%h\n' | sed 's|^packages/||' | sort) + + echo "Found integration packages:" + echo "$package_dirs" + + packages_json=$(echo "$package_dirs" | jq -R -s -c 'split("\n")[:-1]') + echo "packages=$packages_json" >> $GITHUB_OUTPUT + + test-package: + needs: [build-wheels, discover-packages] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + package: ${{ fromJson(needs.discover-packages.outputs.packages) }} + python-version: [ "3.11", "3.12", "3.13" ] + os: [ ubuntu-latest, windows-latest ] + + name: "${{ matrix.package }} / py${{ matrix.python-version }} / ${{ matrix.os }}" + permissions: + contents: read + + steps: + - name: Setup uv + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 + + - name: Setup Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Download wheels + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: uipath-wheels + path: wheels + + - name: Checkout uipath-integrations-python + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + repository: 'UiPath/uipath-integrations-python' + path: 'uipath-integrations-python' + + - name: Checkout uipath-python scripts + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + path: _scripts + sparse-checkout: .github/scripts + + - name: Override uipath packages with local wheels + shell: bash + run: | + PYBIN=$(command -v python || command -v python3) + "$PYBIN" "$GITHUB_WORKSPACE/_scripts/.github/scripts/write_uv_overrides.py" + + - name: Install dependencies and run tests + shell: bash + working-directory: uipath-integrations-python/packages/${{ matrix.package }} + run: | + uv sync + if [ -d tests ]; then + uv run pytest + else + echo "No tests directory found in ${{ matrix.package }}, skipping pytest" + fi + + discover-testcases: + needs: [test-package, discover-packages] + runs-on: uipath-ubuntu-latest + permissions: + contents: read + outputs: + matrix: ${{ steps.discover.outputs.matrix }} + has_testcases: ${{ steps.discover.outputs.has_testcases }} + steps: + - name: Checkout uipath-integrations-python + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + repository: 'UiPath/uipath-integrations-python' + path: 'uipath-integrations-python' + + - name: Discover testcases across packages + id: discover + working-directory: uipath-integrations-python + run: | + # For each package with a testcases/ directory, list its testcase folders + # and emit one matrix entry per (package, testcase) pair. + entries="[]" + for pkg_dir in packages/*/; do + pkg=$(basename "$pkg_dir") + tc_dir="$pkg_dir/testcases" + if [ ! -d "$tc_dir" ]; then + continue + fi + testcases=$(find "$tc_dir" -maxdepth 1 -type d -name "*-*" -printf '%f\n' | sort) + if [ -z "$testcases" ]; then + continue + fi + for tc in $testcases; do + entries=$(echo "$entries" | jq --arg p "$pkg" --arg t "$tc" '. + [{package: $p, testcase: $t}]') + done + done + + echo "Discovered testcase matrix:" + echo "$entries" | jq . + + count=$(echo "$entries" | jq 'length') + if [ "$count" -eq 0 ]; then + echo "has_testcases=false" >> $GITHUB_OUTPUT + echo "matrix=[]" >> $GITHUB_OUTPUT + else + echo "has_testcases=true" >> $GITHUB_OUTPUT + echo "matrix=$(echo "$entries" | jq -c .)" >> $GITHUB_OUTPUT + fi + + run-integration-tests: + needs: [build-wheels, discover-testcases] + if: needs.discover-testcases.outputs.has_testcases == 'true' + runs-on: uipath-ubuntu-latest + container: + image: ghcr.io/astral-sh/uv:python3.12-bookworm + env: + UIPATH_JOB_KEY: "3a03d5cb-fa21-4021-894d-a8e2eda0afe0" + UIPATH_TRACING_ENABLED: false + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.discover-testcases.outputs.matrix) }} + environment: [alpha, staging] # temporary disable [cloud] + + name: "${{ matrix.package }} / ${{ matrix.testcase }} / ${{ matrix.environment }}" + + steps: + - name: Download wheels + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: uipath-wheels + path: wheels + + - name: Checkout uipath-integrations-python + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + repository: 'UiPath/uipath-integrations-python' + path: 'uipath-integrations-python' + + - name: Checkout uipath-python scripts + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + path: _scripts + sparse-checkout: .github/scripts + + - name: Override uipath packages with local wheels + shell: bash + run: | + PYBIN=$(command -v python || command -v python3) + "$PYBIN" "$GITHUB_WORKSPACE/_scripts/.github/scripts/write_uv_overrides.py" + + - name: Install dependencies + working-directory: uipath-integrations-python/packages/${{ matrix.package }} + run: uv sync + + - name: Run testcase + env: + CLIENT_ID: ${{ matrix.environment == 'alpha' && secrets.ALPHA_TEST_CLIENT_ID || matrix.environment == 'staging' && secrets.STAGING_TEST_CLIENT_ID || matrix.environment == 'cloud' && secrets.CLOUD_TEST_CLIENT_ID }} + CLIENT_SECRET: ${{ matrix.environment == 'alpha' && secrets.ALPHA_TEST_CLIENT_SECRET || matrix.environment == 'staging' && secrets.STAGING_TEST_CLIENT_SECRET || matrix.environment == 'cloud' && secrets.CLOUD_TEST_CLIENT_SECRET }} + BASE_URL: ${{ matrix.environment == 'alpha' && secrets.ALPHA_BASE_URL || matrix.environment == 'staging' && secrets.STAGING_BASE_URL || matrix.environment == 'cloud' && secrets.CLOUD_BASE_URL }} + UV_PYTHON: "3.12" + working-directory: uipath-integrations-python/packages/${{ matrix.package }}/testcases/${{ matrix.testcase }} + run: | + echo "Package: ${{ matrix.package }}" + echo "Testcase: ${{ matrix.testcase }}" + echo "Environment: ${{ matrix.environment }}" + + bash run.sh + bash ../common/validate_output.sh + + notify-on-failure: + needs: [test-package, run-integration-tests] + if: always() && contains(github.event.pull_request.labels.*.name, 'test:uipath-integrations') && (needs.test-package.result == 'failure' || needs.run-integration-tests.result == 'failure') + runs-on: uipath-ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Comment on PR + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + with: + script: | + const marker = ''; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const body = [ + marker, + '## :rotating_light: **Heads up: `uipath-integrations` cross-tests are FAILING** :rotating_light:', + '', + 'Your changes may break one or more integrations in **[`uipath-integrations-python`](https://github.com/UiPath/uipath-integrations-python)**:', + '', + '- `uipath-openai-agents`', + '- `uipath-google-adk`', + '- `uipath-agent-framework`', + '- `uipath-llamaindex`', + '- `uipath-pydantic-ai`', + '', + '> :warning: **These checks are NOT enforced by branch protection rules.** Please review the failures before merging.', + '', + `**:mag: [Inspect the failed run →](${runUrl})**`, + ].join('\n'); + + // Delete any prior failure comments for this workflow so the new + // one always lands at the bottom of the PR conversation. + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + for (const c of comments) { + if (c.body && c.body.includes(marker)) { + await github.rest.issues.deleteComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: c.id, + }); + } + } + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); diff --git a/.github/workflows/test-uipath-langchain.yml b/.github/workflows/test-uipath-langchain.yml index f74135975..475efc2e1 100644 --- a/.github/workflows/test-uipath-langchain.yml +++ b/.github/workflows/test-uipath-langchain.yml @@ -6,19 +6,19 @@ on: jobs: build-wheels: - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest permissions: contents: read if: contains(github.event.pull_request.labels.*.name, 'test:uipath-langchain') steps: - name: Checkout uipath-python - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Setup uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" @@ -35,7 +35,7 @@ jobs: run: uv build - name: Upload wheels - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: uipath-wheels path: packages/*/dist/*.whl @@ -53,32 +53,36 @@ jobs: steps: - name: Setup uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} - name: Download wheels - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: uipath-wheels path: wheels - name: Checkout uipath-langchain-python - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: repository: 'UiPath/uipath-langchain-python' path: 'uipath-langchain-python' - - name: Update uipath packages + - name: Checkout uipath-python scripts + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + path: _scripts + sparse-checkout: .github/scripts + + - name: Override uipath packages with local wheels shell: bash - working-directory: uipath-langchain-python run: | - uv add ../wheels/uipath-core/dist/*.whl --dev - uv add ../wheels/uipath-platform/dist/*.whl --dev - uv add ../wheels/uipath/dist/*.whl --dev + PYBIN=$(command -v python || command -v python3) + "$PYBIN" "$GITHUB_WORKSPACE/_scripts/.github/scripts/write_uv_overrides.py" - name: Run uipath-langchain tests working-directory: uipath-langchain-python @@ -87,7 +91,7 @@ jobs: uv run pytest discover-testcases: - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest permissions: contents: read needs: [test-uipath-langchain] @@ -95,7 +99,7 @@ jobs: testcases: ${{ steps.discover.outputs.testcases }} steps: - name: Checkout uipath-langchain-python - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: repository: 'UiPath/uipath-langchain-python' path: 'uipath-langchain-python' @@ -115,7 +119,7 @@ jobs: echo "testcases=$testcases_json" >> $GITHUB_OUTPUT run-uipath-langchain-integration-tests: - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest needs: [build-wheels, discover-testcases] container: image: ghcr.io/astral-sh/uv:python3.12-bookworm @@ -135,24 +139,28 @@ jobs: steps: - name: Download wheels - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: uipath-wheels path: wheels - name: Checkout uipath-langchain-python - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: repository: 'UiPath/uipath-langchain-python' path: 'uipath-langchain-python' - - name: Update uipath packages + - name: Checkout uipath-python scripts + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + path: _scripts + sparse-checkout: .github/scripts + + - name: Override uipath packages with local wheels shell: bash - working-directory: uipath-langchain-python run: | - uv add ../wheels/uipath-core/dist/*.whl - uv add ../wheels/uipath-platform/dist/*.whl - uv add ../wheels/uipath/dist/*.whl + PYBIN=$(command -v python || command -v python3) + "$PYBIN" "$GITHUB_WORKSPACE/_scripts/.github/scripts/write_uv_overrides.py" - name: Install dependencies working-directory: uipath-langchain-python @@ -175,3 +183,51 @@ jobs: # Execute the testcase run script directly bash run.sh bash ../common/validate_output.sh + + notify-on-failure: + needs: [test-uipath-langchain, run-uipath-langchain-integration-tests] + if: always() && contains(github.event.pull_request.labels.*.name, 'test:uipath-langchain') && (needs.test-uipath-langchain.result == 'failure' || needs.run-uipath-langchain-integration-tests.result == 'failure') + runs-on: uipath-ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Comment on PR + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + with: + script: | + const marker = ''; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const body = [ + marker, + '## :rotating_light: **Heads up: `uipath-langchain` cross-tests are FAILING** :rotating_light:', + '', + 'Your changes may break the **[`uipath-langchain-python`](https://github.com/UiPath/uipath-langchain-python)** integration.', + '', + '> :warning: **These checks are NOT enforced by branch protection rules.** Please review the failures before merging.', + '', + `**:mag: [Inspect the failed run →](${runUrl})**`, + ].join('\n'); + + // Delete any prior failure comments for this workflow so the new + // one always lands at the bottom of the PR conversation. + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + for (const c of comments) { + if (c.body && c.body.includes(marker)) { + await github.rest.issues.deleteComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: c.id, + }); + } + } + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); diff --git a/.github/workflows/test-uipath-llamaindex.yml b/.github/workflows/test-uipath-llamaindex.yml deleted file mode 100644 index fcf8d0fb4..000000000 --- a/.github/workflows/test-uipath-llamaindex.yml +++ /dev/null @@ -1,173 +0,0 @@ -name: uipath - Test LlamaIndex - -on: - pull_request: - types: [ opened, synchronize, reopened, labeled ] - -jobs: - build-wheels: - runs-on: ubuntu-latest - permissions: - contents: read - if: contains(github.event.pull_request.labels.*.name, 'test:uipath-llamaindex') - steps: - - name: Checkout uipath-python - uses: actions/checkout@v4 - - - name: Setup uv - uses: astral-sh/setup-uv@v5 - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Build uipath-core package - working-directory: packages/uipath-core - run: uv build - - - name: Build uipath-platform package - working-directory: packages/uipath-platform - run: uv build - - - name: Build uipath package - working-directory: packages/uipath - run: uv build - - - name: Upload wheels - uses: actions/upload-artifact@v4 - with: - name: uipath-wheels - path: packages/*/dist/*.whl - - test-uipath-llamaindex: - needs: [build-wheels] - runs-on: ${{ matrix.os }} - strategy: - matrix: - python-version: [ "3.11", "3.12", "3.13" ] - os: [ ubuntu-latest, windows-latest ] - - permissions: - contents: read - - steps: - - name: Setup uv - uses: astral-sh/setup-uv@v5 - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Download wheels - uses: actions/download-artifact@v4 - with: - name: uipath-wheels - path: wheels - - - name: Checkout uipath-integrations-python - uses: actions/checkout@v4 - with: - repository: 'UiPath/uipath-integrations-python' - path: 'uipath-integrations-python' - - - name: Update uipath packages - shell: bash - working-directory: uipath-integrations-python/packages/uipath-llamaindex - run: | - uv add ../../../wheels/uipath-core/dist/*.whl --dev - uv add ../../../wheels/uipath-platform/dist/*.whl --dev - uv add ../../../wheels/uipath/dist/*.whl --dev - - - name: Run uipath-llamaindex tests - working-directory: uipath-integrations-python/packages/uipath-llamaindex - run: | - uv sync - uv run pytest - - discover-testcases: - runs-on: ubuntu-latest - permissions: - contents: read - needs: [test-uipath-llamaindex] - outputs: - testcases: ${{ steps.discover.outputs.testcases }} - steps: - - name: Checkout uipath-integrations-python - uses: actions/checkout@v4 - with: - repository: 'UiPath/uipath-integrations-python' - path: 'uipath-integrations-python' - - - name: Discover testcases - id: discover - working-directory: uipath-integrations-python/packages/uipath-llamaindex - run: | - # Find all testcase folders (excluding common folders like README, etc.) - testcase_dirs=$(find testcases -maxdepth 1 -type d -name "*-*" | sed 's|testcases/||' | sort) - - echo "Found testcase directories:" - echo "$testcase_dirs" - - # Convert to JSON array for matrix - testcases_json=$(echo "$testcase_dirs" | jq -R -s -c 'split("\n")[:-1]') - echo "testcases=$testcases_json" >> $GITHUB_OUTPUT - - run-uipath-llamaindex-integration-tests: - runs-on: ubuntu-latest - needs: [build-wheels, discover-testcases] - container: - image: ghcr.io/astral-sh/uv:python3.12-bookworm - env: - UIPATH_JOB_KEY: "3a03d5cb-fa21-4021-894d-a8e2eda0afe0" - UIPATH_TRACING_ENABLED: false - permissions: - contents: read - strategy: - fail-fast: false - matrix: - testcase: ${{ fromJson(needs.discover-testcases.outputs.testcases) }} - environment: [alpha, staging] # temporary disable [cloud] - - name: "${{ matrix.testcase }} / ${{ matrix.environment }}" - - steps: - - name: Download wheels - uses: actions/download-artifact@v4 - with: - name: uipath-wheels - path: wheels - - - name: Checkout uipath-integrations-python - uses: actions/checkout@v4 - with: - repository: 'UiPath/uipath-integrations-python' - path: 'uipath-integrations-python' - - - name: Update uipath packages - shell: bash - working-directory: uipath-integrations-python/packages/uipath-llamaindex - run: | - uv add ../../../wheels/uipath-core/dist/*.whl - uv add ../../../wheels/uipath-platform/dist/*.whl - uv add ../../../wheels/uipath/dist/*.whl - - - name: Install dependencies - working-directory: uipath-integrations-python/packages/uipath-llamaindex - run: uv sync - - - name: Run testcase - env: - CLIENT_ID: ${{ matrix.environment == 'alpha' && secrets.ALPHA_TEST_CLIENT_ID || matrix.environment == 'staging' && secrets.STAGING_TEST_CLIENT_ID || matrix.environment == 'cloud' && secrets.CLOUD_TEST_CLIENT_ID }} - CLIENT_SECRET: ${{ matrix.environment == 'alpha' && secrets.ALPHA_TEST_CLIENT_SECRET || matrix.environment == 'staging' && secrets.STAGING_TEST_CLIENT_SECRET || matrix.environment == 'cloud' && secrets.CLOUD_TEST_CLIENT_SECRET }} - BASE_URL: ${{ matrix.environment == 'alpha' && secrets.ALPHA_BASE_URL || matrix.environment == 'staging' && secrets.STAGING_BASE_URL || matrix.environment == 'cloud' && secrets.CLOUD_BASE_URL }} - UV_PYTHON: "3.12" - working-directory: uipath-integrations-python/packages/uipath-llamaindex/testcases/${{ matrix.testcase }} - run: | - echo "Running testcase: ${{ matrix.testcase }}" - echo "Environment: ${{ matrix.environment }}" - - # Execute the testcase run script directly - bash run.sh - bash ../common/validate_output.sh diff --git a/.github/workflows/test-uipath-runtime.yml b/.github/workflows/test-uipath-runtime.yml index 13ad019ef..bb72c2f97 100644 --- a/.github/workflows/test-uipath-runtime.yml +++ b/.github/workflows/test-uipath-runtime.yml @@ -6,19 +6,19 @@ on: jobs: build-wheels: - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest permissions: contents: read if: contains(github.event.pull_request.labels.*.name, 'test:uipath-runtime') steps: - name: Checkout uipath-python - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Setup uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" @@ -27,7 +27,7 @@ jobs: run: uv build - name: Upload wheels - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: uipath-core-wheel path: packages/uipath-core/dist/*.whl @@ -45,29 +45,36 @@ jobs: steps: - name: Setup uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} - name: Download wheels - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: uipath-core-wheel path: wheels - name: Checkout uipath-runtime-python - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: repository: 'UiPath/uipath-runtime-python' path: 'uipath-runtime-python' - - name: Update uipath-core version + - name: Checkout uipath-python scripts + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + path: _scripts + sparse-checkout: .github/scripts + + - name: Override uipath packages with local wheels shell: bash - working-directory: uipath-runtime-python - run: uv add ../wheels/*.whl --dev + run: | + PYBIN=$(command -v python || command -v python3) + "$PYBIN" "$GITHUB_WORKSPACE/_scripts/.github/scripts/write_uv_overrides.py" - name: Run uipath-runtime tests working-directory: uipath-runtime-python diff --git a/SETUP.MD b/SETUP.MD new file mode 100644 index 000000000..4e728471b --- /dev/null +++ b/SETUP.MD @@ -0,0 +1,133 @@ +# SETUP.MD + +This file documents how to provision a clean development environment for the three packages in this repo (`uipath-core`, `uipath-platform`, `uipath`), run the build, execute the tests, and validate a sample code change end-to-end. It is intended both as a quick reference for human contributors and as a structured guide for automated environment-setup tooling. + +## Prerequisites + +- Python 3.11+ +- [uv](https://docs.astral.sh/uv/) 0.5+ + +### Supported platforms + +`uv` is shell- and OS-agnostic, so the commands below run unchanged on every supported platform: + +- [x] Linux +- [x] Windows +- [x] macOS + +## Environment Variables + +None required for environment setup, build, or unit tests. The suites under the `Test` section run fully offline and require no external authentication. + +> **All commands below must be run from the repository root.** The `uv --directory packages/` invocations resolve each subpackage relative to the current working directory. The first line of `## Setup` enforces this by `cd`-ing to the git root. + +## Setup + +```bash +cd "$(git rev-parse --show-toplevel)" +python3 -m pip install --upgrade uv + +# Sync all three packages (dependency order: core → platform → main) +uv --directory packages/uipath-core sync --all-extras +uv --directory packages/uipath-platform sync --all-extras +uv --directory packages/uipath sync --all-extras +``` + +## Verify Setup + +```bash +uv --version +uv --directory packages/uipath-core run python --version +uv --directory packages/uipath-core run python -c "import uipath.core; print('uipath-core ok')" +uv --directory packages/uipath-platform run python -c "import uipath.platform; print('uipath-platform ok')" +uv --directory packages/uipath run python -c "import uipath; print('uipath ok')" +``` + +## Build + +N/A + +## Test + +```bash +uv --directory packages/uipath-core run pytest +uv --directory packages/uipath-platform run pytest +uv --directory packages/uipath run pytest +``` + +> Note: `uipath-platform`'s `pyproject.toml` already excludes its E2E tests via `addopts = "... -m 'not e2e'"`. `uipath-core` and `uipath` do not register an `e2e` marker. + +## Sample Code Change + +### The change + +Add a new `size` property to `SpanRegistry` in `packages/uipath-core/src/uipath/core/tracing/span_utils.py`, immediately after the `clear` method and before the `# Global span registry instance` comment: + +```python +@property +def size(self) -> int: + """Return the number of currently registered spans.""" + return len(self._spans) +``` + +Then create `packages/uipath-core/tests/tracing/test_span_registry_size.py` with two pytest tests: + +```python +from unittest.mock import MagicMock + +from uipath.core.tracing.span_utils import SpanRegistry + + +def _make_span(span_id: int) -> MagicMock: + span = MagicMock() + span.get_span_context.return_value.span_id = span_id + span.parent = None # registered as a root span (no parent) + return span + + +def test_size_empty_registry() -> None: + registry = SpanRegistry() + assert registry.size == 0 + + +def test_size_after_registrations() -> None: + registry = SpanRegistry() + registry.register_span(_make_span(1)) + registry.register_span(_make_span(2)) + assert registry.size == 2 +``` + +### Verification + +```bash +uv --directory packages/uipath-core run pytest tests/tracing/test_span_registry_size.py -v +``` + +## Test with a real UiPath Coded Agent + +> This section is for human contributors who want to validate changes end-to-end against the real cloud platform. It is **not executed by the Agentic Inner Loop validation pipeline** — that pipeline only runs the sections above (Setup → Verify → Build → Test → Sample Code Change). + +The unit tests above are necessary but not sufficient — they don't exercise the package end-to-end through a real agent. The flow below validates changes against a live runtime: + +1. Apply the code changes locally. +2. Run the unit tests (see the `Sample Code Change` section above). +3. Scaffold a coded UiPath agent that exercises the changed code path. +4. In the downstream project's `pyproject.toml`, add this local library as an editable dependency (substitute `uipath`, `uipath-platform`, or `uipath-core` depending on which package you changed): + + ```toml + [tool.uv.sources] + uipath = { path = "../path/to/uipath-python/packages/uipath", editable = true } + ``` + +5. Exercise the new behavior end-to-end: + + ```bash + uv run uipath run --input '{...}' + ``` + +6. (Optional) Open a PR and apply the `build:dev` label — this publishes the development version to Test PyPI. +7. The PR description is updated automatically with instructions for pointing the downstream agent at the Test PyPI dev version. +8. Validate the new behavior against the real platform — use either or both of the deploy targets below (Studio Web and Orchestrator are not mutually exclusive): + - **Studio Web**: export the `UIPATH_PROJECT_ID` environment variable pointing to an existing Coded Agent project in your solution, then run [`uipath push`](https://uipath.github.io/uipath-python/cli/#push) to push the dev version to that project. Open it in Studio Web and exercise the changed code path. + - **Orchestrator**: run [`uipath deploy`](https://uipath.github.io/uipath-python/cli/#deploy) to deploy the dev version as a package, then start a job in Orchestrator and exercise the changed code path. +9. Once validation is done, close the dev PR — these PRs are not meant to be merged; their only purpose was to publish a Test PyPI build for end-to-end validation. diff --git a/packages/uipath-core/pyproject.toml b/packages/uipath-core/pyproject.toml index 5604e3938..dc2570992 100644 --- a/packages/uipath-core/pyproject.toml +++ b/packages/uipath-core/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath-core" -version = "0.5.10" +version = "0.5.31" description = "UiPath Core abstractions" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" @@ -95,15 +95,33 @@ warn_required_dynamic_aliases = true [tool.pytest.ini_options] testpaths = ["tests"] python_files = "test_*.py" -addopts = "-ra -q --cov=src/uipath --cov-report=term-missing" +addopts = "-ra -q --cov=src --cov-report=term-missing" asyncio_default_fixture_loop_scope = "function" asyncio_mode = "auto" +[tool.coverage.run] +source = ["src"] +relative_files = true +omit = [ + "*/tests/*", + "*/__pycache__/*", + "*/site-packages/*", + "*/conftest.py", +] + [tool.coverage.report] show_missing = true +precision = 2 +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "raise NotImplementedError", + "if TYPE_CHECKING:", + "@(abc\\.)?abstractmethod", +] -[tool.coverage.run] -source = ["src"] +[tool.uv] +exclude-newer = "2 days" [[tool.uv.index]] name = "testpypi" diff --git a/packages/uipath-core/src/uipath/core/adapters/__init__.py b/packages/uipath-core/src/uipath/core/adapters/__init__.py new file mode 100644 index 000000000..c3675b404 --- /dev/null +++ b/packages/uipath-core/src/uipath/core/adapters/__init__.py @@ -0,0 +1,20 @@ +"""Generic adapter contracts for framework integrations. + +This package holds only the abstract contracts — concrete adapter +implementations live in framework-specific plugin packages (e.g. +``uipath-langchain``, ``uipath-openai``). A framework plugin is the one +that knows its own native wiring seam (callback handler list, hook +registry, …) and installs governance there directly; uipath-core only +defines the protocol an evaluator must satisfy. + +Public surface: + +- :class:`EvaluatorProtocol` – structural protocol the framework + plugin expects from any policy evaluator. +""" + +from .evaluator import EvaluatorProtocol + +__all__ = [ + "EvaluatorProtocol", +] diff --git a/packages/uipath-core/src/uipath/core/adapters/evaluator.py b/packages/uipath-core/src/uipath/core/adapters/evaluator.py new file mode 100644 index 000000000..4f25097c5 --- /dev/null +++ b/packages/uipath-core/src/uipath/core/adapters/evaluator.py @@ -0,0 +1,96 @@ +"""Structural contract for the policy evaluator a framework plugin talks to. + +Framework plugins call into a policy evaluator at each lifecycle hook. +Concrete evaluator implementations (the native runtime evaluator, a +Microsoft AGT bridge, a composite, …) live in packages outside +``uipath-core`` — plugins depend only on this structural protocol so +they can be swapped against any of them without code change. + +``EvaluatorProtocol`` is a :class:`typing.Protocol` so any class whose +methods match the signatures below satisfies the contract without +inheritance. +""" + +from __future__ import annotations + +from typing import Any, Protocol, runtime_checkable + +from uipath.core.governance.models import AuditRecord + + +@runtime_checkable +class EvaluatorProtocol(Protocol): + """Structural protocol a framework plugin expects from a policy evaluator. + + Every ``evaluate_*`` method returns an :class:`AuditRecord` — the + per-hook audit envelope holding the per-rule + :class:`RuleEvaluation` list, the final action, and the trace / + agent metadata. Callers get a typed result; no downcasting is + required. + """ + + def evaluate_before_agent( + self, + agent_input: str, + agent_name: str, + runtime_id: str, + model_name: str = "", + **kwargs: Any, + ) -> AuditRecord: + """Evaluate BEFORE_AGENT rules.""" + ... + + def evaluate_after_agent( + self, + agent_output: str, + agent_name: str, + runtime_id: str, + **kwargs: Any, + ) -> AuditRecord: + """Evaluate AFTER_AGENT rules.""" + ... + + def evaluate_before_model( + self, + model_input: str, + agent_name: str, + runtime_id: str, + messages: list[dict[str, Any]] | None = None, + model_name: str = "", + **kwargs: Any, + ) -> AuditRecord: + """Evaluate BEFORE_MODEL rules.""" + ... + + def evaluate_after_model( + self, + model_output: str, + agent_name: str, + runtime_id: str, + **kwargs: Any, + ) -> AuditRecord: + """Evaluate AFTER_MODEL rules.""" + ... + + def evaluate_tool_call( + self, + tool_name: str, + tool_args: dict[str, Any], + agent_name: str, + runtime_id: str, + session_state: dict[str, Any] | None = None, + **kwargs: Any, + ) -> AuditRecord: + """Evaluate TOOL_CALL rules.""" + ... + + def evaluate_after_tool( + self, + tool_name: str, + tool_result: str, + agent_name: str, + runtime_id: str, + **kwargs: Any, + ) -> AuditRecord: + """Evaluate AFTER_TOOL rules.""" + ... diff --git a/packages/uipath-core/src/uipath/core/chat/__init__.py b/packages/uipath-core/src/uipath/core/chat/__init__.py index 476cb9352..ee4a4c674 100644 --- a/packages/uipath-core/src/uipath/core/chat/__init__.py +++ b/packages/uipath-core/src/uipath/core/chat/__init__.py @@ -71,6 +71,7 @@ ) from .event import UiPathConversationEvent, UiPathConversationLabelUpdatedEvent from .exchange import ( + UiPathClientSideToolDeclaration, UiPathConversationExchange, UiPathConversationExchangeData, UiPathConversationExchangeEndEvent, @@ -107,13 +108,22 @@ UiPathSessionStartEvent, ) from .tool import ( + UiPathConversationExecutingToolCallEvent, UiPathConversationToolCall, + UiPathConversationToolCallConfirmation, + UiPathConversationToolCallConfirmationData, + UiPathConversationToolCallConfirmationEvent, UiPathConversationToolCallData, UiPathConversationToolCallEndEvent, UiPathConversationToolCallEvent, UiPathConversationToolCallResult, UiPathConversationToolCallStartEvent, ) +from .voice import ( + UiPathVoiceToolCallMessage, + UiPathVoiceToolCallRequest, + UiPathVoiceToolCallResult, +) __all__ = [ # Root @@ -130,6 +140,7 @@ "UiPathSessionEndingEvent", "UiPathSessionEndEvent", # Exchange + "UiPathClientSideToolDeclaration", "UiPathConversationExchangeStartEvent", "UiPathConversationExchangeEndEvent", "UiPathConversationExchangeEvent", @@ -141,19 +152,6 @@ "UiPathConversationMessageEvent", "UiPathConversationMessageData", "UiPathConversationMessage", - # Interrupt - "InterruptTypeEnum", - "UiPathConversationInterruptStartEvent", - "UiPathConversationInterruptEndEvent", - "UiPathConversationInterruptEvent", - "UiPathConversationToolCallConfirmationValue", - "UiPathConversationToolCallConfirmationEndValue", - "UiPathConversationToolCallConfirmationInterruptStartEvent", - "UiPathConversationToolCallConfirmationInterruptEndEvent", - "UiPathConversationGenericInterruptStartEvent", - "UiPathConversationGenericInterruptEndEvent", - "UiPathConversationInterruptData", - "UiPathConversationInterrupt", # Content "UiPathConversationContentPartChunkEvent", "UiPathConversationContentPartStartEvent", @@ -176,8 +174,12 @@ "UiPathConversationCitationData", "UiPathConversationCitation", # Tool + "UiPathConversationExecutingToolCallEvent", "UiPathConversationToolCallStartEvent", "UiPathConversationToolCallEndEvent", + "UiPathConversationToolCallConfirmation", + "UiPathConversationToolCallConfirmationData", + "UiPathConversationToolCallConfirmationEvent", "UiPathConversationToolCallEvent", "UiPathConversationToolCallResult", "UiPathConversationToolCallData", @@ -189,4 +191,21 @@ "UiPathConversationAsyncInputStreamEvent", # Meta "UiPathConversationMetaEvent", + # Voice + "UiPathVoiceToolCallRequest", + "UiPathVoiceToolCallMessage", + "UiPathVoiceToolCallResult", + # Interrupt (compat shims — deprecated, see interrupt.py) + "InterruptTypeEnum", + "UiPathConversationInterruptStartEvent", + "UiPathConversationInterruptEndEvent", + "UiPathConversationInterruptEvent", + "UiPathConversationInterruptData", + "UiPathConversationInterrupt", + "UiPathConversationGenericInterruptStartEvent", + "UiPathConversationGenericInterruptEndEvent", + "UiPathConversationToolCallConfirmationValue", + "UiPathConversationToolCallConfirmationEndValue", + "UiPathConversationToolCallConfirmationInterruptStartEvent", + "UiPathConversationToolCallConfirmationInterruptEndEvent", ] diff --git a/packages/uipath-core/src/uipath/core/chat/content.py b/packages/uipath-core/src/uipath/core/chat/content.py index cc6300490..38ad2bbd6 100644 --- a/packages/uipath-core/src/uipath/core/chat/content.py +++ b/packages/uipath-core/src/uipath/core/chat/content.py @@ -2,6 +2,7 @@ from __future__ import annotations +import uuid from typing import Any, Sequence from pydantic import BaseModel, ConfigDict, Field @@ -95,7 +96,8 @@ class UiPathConversationContentPartData(BaseModel): mime_type: str = Field(..., alias="mimeType") data: InlineOrExternal - citations: Sequence[UiPathConversationCitationData] + metadata: dict[str, Any] | None = Field(None, alias="metaData") + citations: Sequence[UiPathConversationCitationData] = Field(default_factory=list) is_transcript: bool | None = Field(None, alias="isTranscript") is_incomplete: bool | None = Field(None, alias="isIncomplete") name: str | None = None @@ -106,11 +108,13 @@ class UiPathConversationContentPartData(BaseModel): class UiPathConversationContentPart(UiPathConversationContentPartData): """Represents a single part of message content.""" - content_part_id: str = Field(..., alias="contentPartId") - created_at: str = Field(..., alias="createdAt") - updated_at: str = Field(..., alias="updatedAt") + content_part_id: str = Field( + default_factory=lambda: str(uuid.uuid4()), alias="contentPartId" + ) + created_at: str | None = Field(None, alias="createdAt") + updated_at: str | None = Field(None, alias="updatedAt") # Override to use full type - citations: Sequence[UiPathConversationCitation] + citations: Sequence[UiPathConversationCitation] = Field(default_factory=list) model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) diff --git a/packages/uipath-core/src/uipath/core/chat/exchange.py b/packages/uipath-core/src/uipath/core/chat/exchange.py index 788bbe560..835489dfb 100644 --- a/packages/uipath-core/src/uipath/core/chat/exchange.py +++ b/packages/uipath-core/src/uipath/core/chat/exchange.py @@ -28,11 +28,24 @@ ) +class UiPathClientSideToolDeclaration(BaseModel): + """A client-side tool declaration from the SDK client.""" + + name: str + input_schema: dict[str, Any] | None = Field(None, alias="inputSchema") + output_schema: dict[str, Any] | None = Field(None, alias="outputSchema") + + model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + + class UiPathConversationExchangeStartEvent(BaseModel): """Signals the start of an exchange of messages within a conversation.""" conversation_sequence: int | None = Field(None, alias="conversationSequence") metadata: dict[str, Any] | None = Field(None, alias="metaData") + client_side_tools: list[UiPathClientSideToolDeclaration] | None = Field( + None, alias="clientSideTools" + ) timestamp: str | None = None model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) diff --git a/packages/uipath-core/src/uipath/core/chat/interrupt.py b/packages/uipath-core/src/uipath/core/chat/interrupt.py index a2ce3e13f..9094d1597 100644 --- a/packages/uipath-core/src/uipath/core/chat/interrupt.py +++ b/packages/uipath-core/src/uipath/core/chat/interrupt.py @@ -1,4 +1,14 @@ -"""Interrupt events for human-in-the-loop patterns.""" +"""Compatibility shims for legacy interrupt event types. + +The interrupt-based tool-call confirmation flow was replaced by `confirmToolCall` +on the tool call event itself (see PR #1558). The original `interrupt.py` was +removed in `uipath-core` 0.5.13, but published `uipath-runtime` versions still +import these names at module load time, breaking installs that pull the new +`uipath-core` alongside an older runtime. + +These shims keep those imports working. They are not used by current code paths +and should be removed in the next minor bump of `uipath-core`. +""" from enum import Enum from typing import Any, Literal, Union @@ -93,7 +103,7 @@ class UiPathConversationInterruptEvent(BaseModel): class UiPathConversationInterruptData(BaseModel): - """Represents the core data of an interrupt within a message - a pause point where the agent needs external input.""" + """Core data of an interrupt within a message.""" type: str interrupt_value: Any = Field(..., alias="interruptValue") @@ -103,7 +113,7 @@ class UiPathConversationInterruptData(BaseModel): class UiPathConversationInterrupt(UiPathConversationInterruptData): - """Represents an interrupt within a message - a pause point where the agent needs external input.""" + """An interrupt within a message — a pause point where the agent needs external input.""" interrupt_id: str = Field(..., alias="interruptId") created_at: str = Field(..., alias="createdAt") diff --git a/packages/uipath-core/src/uipath/core/chat/message.py b/packages/uipath-core/src/uipath/core/chat/message.py index 48e79171f..37aa2bd76 100644 --- a/packages/uipath-core/src/uipath/core/chat/message.py +++ b/packages/uipath-core/src/uipath/core/chat/message.py @@ -1,5 +1,6 @@ """Message-level events.""" +import uuid from typing import Any, Sequence from pydantic import BaseModel, ConfigDict, Field @@ -10,11 +11,6 @@ UiPathConversationContentPartEvent, ) from .error import UiPathConversationErrorEvent -from .interrupt import ( - UiPathConversationInterrupt, - UiPathConversationInterruptData, - UiPathConversationInterruptEvent, -) from .tool import ( UiPathConversationToolCall, UiPathConversationToolCallData, @@ -53,7 +49,6 @@ class UiPathConversationMessageEvent(BaseModel): None, alias="contentPart" ) tool_call: UiPathConversationToolCallEvent | None = Field(None, alias="toolCall") - interrupt: UiPathConversationInterruptEvent | None = None meta_event: dict[str, Any] | None = Field(None, alias="metaEvent") error: UiPathConversationErrorEvent | None = Field(None, alias="messageError") @@ -67,8 +62,9 @@ class UiPathConversationMessageData(BaseModel): content_parts: Sequence[UiPathConversationContentPartData] = Field( ..., alias="contentParts" ) - tool_calls: Sequence[UiPathConversationToolCallData] = Field(..., alias="toolCalls") - interrupts: Sequence[UiPathConversationInterruptData] + tool_calls: Sequence[UiPathConversationToolCallData] = Field( + default_factory=list, alias="toolCalls" + ) model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) @@ -76,16 +72,19 @@ class UiPathConversationMessageData(BaseModel): class UiPathConversationMessage(UiPathConversationMessageData): """Represents a single message within an exchange.""" - message_id: str = Field(..., alias="messageId") - created_at: str = Field(..., alias="createdAt") - updated_at: str = Field(..., alias="updatedAt") + message_id: str = Field( + default_factory=lambda: str(uuid.uuid4()), alias="messageId" + ) + created_at: str | None = Field(None, alias="createdAt") + updated_at: str | None = Field(None, alias="updatedAt") span_id: str | None = Field(None, alias="spanId") # Overrides to use full types content_parts: Sequence[UiPathConversationContentPart] = Field( ..., alias="contentParts" ) - tool_calls: Sequence[UiPathConversationToolCall] = Field(..., alias="toolCalls") - interrupts: Sequence[UiPathConversationInterrupt] + tool_calls: Sequence[UiPathConversationToolCall] = Field( + default_factory=list, alias="toolCalls" + ) model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) diff --git a/packages/uipath-core/src/uipath/core/chat/tool.py b/packages/uipath-core/src/uipath/core/chat/tool.py index 9c9e911bd..514e42908 100644 --- a/packages/uipath-core/src/uipath/core/chat/tool.py +++ b/packages/uipath-core/src/uipath/core/chat/tool.py @@ -25,6 +25,10 @@ class UiPathConversationToolCallStartEvent(BaseModel): timestamp: str | None = None input: dict[str, Any] | None = None metadata: dict[str, Any] | None = Field(None, alias="metaData") + require_confirmation: bool | None = Field(None, alias="requireConfirmation") + input_schema: Any | None = Field(None, alias="inputSchema") + is_client_side_tool: bool | None = Field(None, alias="isClientSideTool") + output_schema: Any | None = Field(None, alias="outputSchema") model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) @@ -41,6 +45,47 @@ class UiPathConversationToolCallEndEvent(BaseModel): model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) +class UiPathConversationExecutingToolCallEvent(BaseModel): + """Signals the client that the tool is about to be executed. + + Emitted in all scenarios. For client-side tools, the client should begin + executing its handler upon receiving this event. + """ + + timestamp: str | None = None + input: dict[str, Any] | None = None + + model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + + +class UiPathConversationToolCallConfirmationEvent(BaseModel): + """Signals a tool call confirmation (approve/reject) from the client.""" + + approved: bool + input: Any | None = None + + model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + + +class UiPathConversationToolCallConfirmationData(BaseModel): + """Represents the core data of a tool call confirmation.""" + + approved: bool + input: Any | None = None + + model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + + +class UiPathConversationToolCallConfirmation( + UiPathConversationToolCallConfirmationData +): + """Represents the stored confirmation state on a tool call.""" + + confirmed_at: str | None = Field(None, alias="confirmedAt") + + model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + + class UiPathConversationToolCallEvent(BaseModel): """Encapsulates the data related to a tool call event.""" @@ -49,6 +94,12 @@ class UiPathConversationToolCallEvent(BaseModel): None, alias="startToolCall" ) end: UiPathConversationToolCallEndEvent | None = Field(None, alias="endToolCall") + confirm: UiPathConversationToolCallConfirmationEvent | None = Field( + None, alias="confirmToolCall" + ) + executing: UiPathConversationExecutingToolCallEvent | None = Field( + None, alias="executingToolCall" + ) meta_event: dict[str, Any] | None = Field(None, alias="metaEvent") error: UiPathConversationErrorEvent | None = Field(None, alias="toolCallError") @@ -61,6 +112,9 @@ class UiPathConversationToolCallData(BaseModel): name: str input: dict[str, Any] | None = None result: UiPathConversationToolCallResult | None = None + require_confirmation: bool | None = Field(None, alias="requireConfirmation") + input_schema: Any | None = Field(None, alias="inputSchema") + confirmation: UiPathConversationToolCallConfirmationData | None = None model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) @@ -72,5 +126,6 @@ class UiPathConversationToolCall(UiPathConversationToolCallData): timestamp: str | None = None created_at: str = Field(..., alias="createdAt") updated_at: str = Field(..., alias="updatedAt") + confirmation: UiPathConversationToolCallConfirmation | None = None model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) diff --git a/packages/uipath-core/src/uipath/core/chat/voice.py b/packages/uipath-core/src/uipath/core/chat/voice.py new file mode 100644 index 000000000..7b1adf7e8 --- /dev/null +++ b/packages/uipath-core/src/uipath/core/chat/voice.py @@ -0,0 +1,30 @@ +"""Voice tool-call wire models (CAS socket.io).""" + +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + + +class _VoiceWire(BaseModel): + model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + + +class UiPathVoiceToolCallRequest(_VoiceWire): + """Single tool call in a batch.""" + + call_id: str = Field(..., alias="callId") + tool_name: str = Field(..., alias="toolName") + args: dict[str, Any] + + +class UiPathVoiceToolCallMessage(_VoiceWire): + """Batch of tool calls from CAS.""" + + calls: list[UiPathVoiceToolCallRequest] = Field(..., min_length=1) + + +class UiPathVoiceToolCallResult(_VoiceWire): + """Result of a single tool call.""" + + result: str + is_error: bool = Field(..., alias="isError") diff --git a/packages/uipath-core/src/uipath/core/governance/__init__.py b/packages/uipath-core/src/uipath/core/governance/__init__.py new file mode 100644 index 000000000..4bf855b82 --- /dev/null +++ b/packages/uipath-core/src/uipath/core/governance/__init__.py @@ -0,0 +1,52 @@ +"""UiPath governance shared contracts. + +Evaluator-agnostic types every governance consumer references — the +runtime layer, adapter packages, and customer code that catches +:class:`GovernanceBlockException`. The full runtime / audit / +native-evaluator implementation lives outside this package; this +core surface is just the contracts. +""" + +from .config import ( + GOVERNANCE_FEATURE_FLAG, + is_governance_enabled, +) +from .exceptions import ( + GovernanceBlockException, + GovernanceConfigError, + GovernanceViolation, + Severity, +) +from .models import Action, AuditRecord, EnforcementMode, LifecycleHook, RuleEvaluation +from .providers import ( + FiredRule, + GovernanceCompensationProvider, + GovernancePolicyProvider, + GovernRequest, + PolicyContext, + PolicyResponse, +) + +__all__ = [ + # Output models (cross adapter boundary) + "Action", + "AuditRecord", + "EnforcementMode", + "LifecycleHook", + "RuleEvaluation", + # Config + "GOVERNANCE_FEATURE_FLAG", + "is_governance_enabled", + # Exceptions + "GovernanceBlockException", + "GovernanceConfigError", + "GovernanceViolation", + "Severity", + # Provider protocols + wire models + "FiredRule", + "GovernanceCompensationProvider", + "GovernancePolicyProvider", + "GovernRequest", + "PolicyContext", + "PolicyResponse", +] diff --git a/packages/uipath-core/src/uipath/core/governance/config.py b/packages/uipath-core/src/uipath/core/governance/config.py new file mode 100644 index 000000000..cbcbd577a --- /dev/null +++ b/packages/uipath-core/src/uipath/core/governance/config.py @@ -0,0 +1,37 @@ +"""Governance configuration. + +Process-level feature-flag gate that decides whether the Python +governance checker runs at all. The +:class:`uipath.core.governance.EnforcementMode` value type is defined +in :mod:`uipath.core.governance.models`; the per-policy runtime state +that selects a mode (backend-supplied via the ``/runtime/policy`` +client) lives outside this package. +""" + +from __future__ import annotations + +from uipath.core.feature_flags import FeatureFlags + +# Feature flag name controlling whether governance runs. +# A single shared gate so the host-driven injection path and direct +# callers (agents constructing an evaluator themselves) honour the +# same toggle. +GOVERNANCE_FEATURE_FLAG = "EnablePythonGovernanceChecker" + + +def is_governance_enabled() -> bool: + """Return whether the ``EnablePythonGovernanceChecker`` flag is enabled. + + Governance is **off by default** — the flag must be explicitly set + to ``true`` (programmatically via the ``FeatureFlags`` registry, or + via the ``UIPATH_FEATURE_EnablePythonGovernanceChecker`` env var) + for this function to return ``True``. + + Resolution order: + + 1. :meth:`uipath.core.feature_flags.FeatureFlagsManager.is_flag_enabled` - + the in-process programmatic registry (typically populated from + gitops) and its own ``UIPATH_FEATURE_`` env-var fallback. + 2. Default ``False`` (governance disabled). + """ + return FeatureFlags.is_flag_enabled(GOVERNANCE_FEATURE_FLAG, default=False) diff --git a/packages/uipath-core/src/uipath/core/governance/exceptions.py b/packages/uipath-core/src/uipath/core/governance/exceptions.py new file mode 100644 index 000000000..48f4b178a --- /dev/null +++ b/packages/uipath-core/src/uipath/core/governance/exceptions.py @@ -0,0 +1,114 @@ +"""Governance exception types.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + +from uipath.core.governance.models import AuditRecord + +_DEFAULT_RULE_ID = "POLICY" +_DEFAULT_RULE_NAME = "Governance Policy" +_MSG_PREFIX = "[Governance Policy Violation]" + + +class Severity(str, Enum): + """Severity classification for a governance violation.""" + + LOW = "low" + MEDIUM = "medium" + HIGH = "high" + CRITICAL = "critical" + + +@dataclass +class GovernanceViolation: + """Details of a governance rule violation.""" + + rule_id: str + rule_name: str + detail: str + severity: Severity = Severity.HIGH + + +def _format_violation_message(rule_id: str, rule_name: str, detail: str) -> str: + return f"{_MSG_PREFIX} {rule_name} ({rule_id}): {detail}" + + +class GovernanceBlockException(Exception): + """Raised when a governance policy blocks an operation. + + This exception indicates that the AI agent's operation was blocked by + a configured governance policy, not an unexpected system error. + + Prefer the classmethod constructors (:meth:`from_violation`, + :meth:`from_audit_record`) when you have structured context — the + default constructor is for raw-message use only. + """ + + # Error code for Orchestrator categorization + error_code: str = "GOVERNANCE_POLICY_VIOLATION" + + def __init__( + self, + message: str | None = None, + *, + violation: GovernanceViolation | None = None, + audit_record: AuditRecord | None = None, + rule_id: str = _DEFAULT_RULE_ID, + rule_name: str = _DEFAULT_RULE_NAME, + ) -> None: + """Construct from a pre-formatted message and optional structured context. + + Most callers should use :meth:`from_violation` or + :meth:`from_audit_record` instead of passing structured context + directly. + """ + self.violation = violation + self.audit_record = audit_record + self.rule_id = rule_id + self.rule_name = rule_name + super().__init__( + message or f"{_MSG_PREFIX} Operation blocked by governance policy." + ) + + @classmethod + def from_violation( + cls, violation: GovernanceViolation + ) -> "GovernanceBlockException": + """Build from a structured :class:`GovernanceViolation`.""" + return cls( + message=_format_violation_message( + violation.rule_id, violation.rule_name, violation.detail + ), + violation=violation, + rule_id=violation.rule_id, + rule_name=violation.rule_name, + ) + + @classmethod + def from_audit_record(cls, audit_record: AuditRecord) -> "GovernanceBlockException": + """Build from an :class:`AuditRecord` — first matched rule wins.""" + matched_rules = [e for e in audit_record.evaluations if e.matched] + if matched_rules: + rule = matched_rules[0] + message = _format_violation_message( + rule.rule_id, rule.rule_name, rule.detail or "Policy violation detected" + ) + return cls( + message=message, + audit_record=audit_record, + rule_id=rule.rule_id, + rule_name=rule.rule_name, + ) + return cls( + message=( + f"{_MSG_PREFIX} Operation blocked. " + f"Rules evaluated: {len(audit_record.evaluations)}" + ), + audit_record=audit_record, + ) + + +class GovernanceConfigError(RuntimeError): + """Raised when governance is misconfigured.""" diff --git a/packages/uipath-core/src/uipath/core/governance/models.py b/packages/uipath-core/src/uipath/core/governance/models.py new file mode 100644 index 000000000..29dccc121 --- /dev/null +++ b/packages/uipath-core/src/uipath/core/governance/models.py @@ -0,0 +1,88 @@ +"""Shared governance contracts. + +Two groups of types live here, both kept free of policy-input concepts +(``Rule``/``Check``/``Condition``) so adapter packages don't inherit +the native policy model: + +- **Output types** (:class:`Action`, :class:`LifecycleHook`, + :class:`RuleEvaluation`, :class:`AuditRecord`) — cross the adapter + boundary at evaluation time: every evaluator implementation (native, + AGT, composite, …) produces them, and every adapter consumes them. +- **Configuration value types** (:class:`EnforcementMode`) — describe + governance configuration shared by core and its consumers. The + per-policy runtime state that selects a mode lives outside this + package; only the value type lives here. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from enum import Enum +from typing import Any + + +class Action(str, Enum): + """Actions that can be taken when a rule matches.""" + + ALLOW = "allow" + DENY = "deny" + AUDIT = "audit" + ESCALATE = "escalate" + + +class LifecycleHook(str, Enum): + """Agent lifecycle hooks where rules can be evaluated.""" + + BEFORE_AGENT = "before_agent" + AFTER_AGENT = "after_agent" + BEFORE_MODEL = "before_model" + AFTER_MODEL = "after_model" + TOOL_CALL = "tool_call" + AFTER_TOOL = "after_tool" + + +class EnforcementMode(str, Enum): + """Governance enforcement modes.""" + + AUDIT = "audit" # Evaluate and log; never block. + ENFORCE = "enforce" # Block on DENY rules. + DISABLED = "disabled" # Skip evaluation entirely. + + +@dataclass +class RuleEvaluation: + """Result of evaluating a single rule.""" + + rule_id: str + rule_name: str + matched: bool + detail: str = "" + pack_name: str = "" + action: Action = Action.ALLOW + description: str = "" + check_results: list[dict[str, Any]] = field(default_factory=list) + + +@dataclass +class AuditRecord: + """Complete audit record for a governance evaluation. + + ``trace_id`` is intentionally absent. Trace correlation is resolved + by the concrete provider at request time (via OpenTelemetry's + native span identity) — per-evaluation trace ids aren't part of + the audit-record contract. + """ + + timestamp: datetime + agent_name: str + runtime_id: str + hook: LifecycleHook + evaluations: list[RuleEvaluation] + final_action: Action + metadata: dict[str, Any] = field(default_factory=dict) + rules_matched: int = field(init=False) + + def __post_init__(self) -> None: + """Derive rules_matched from the evaluations list.""" + self.rules_matched = sum(1 for e in self.evaluations if e.matched) diff --git a/packages/uipath-core/src/uipath/core/governance/providers.py b/packages/uipath-core/src/uipath/core/governance/providers.py new file mode 100644 index 000000000..5435ad389 --- /dev/null +++ b/packages/uipath-core/src/uipath/core/governance/providers.py @@ -0,0 +1,182 @@ +"""Provider protocols for governance backend interactions. + +The runtime needs two backend interactions to function: + +- Fetching the policy pack at startup. +- Firing the compensating ``/runtime/govern`` POST when a + ``guardrail_fallback`` rule matches so the server can run the disabled + centralised guardrail and write the per-rule LLMOps audit records. + +Both have wire formats owned by the ``agenticgovernance_`` ingress. +Defining the contracts here — alongside :class:`EvaluatorProtocol` — +lets runtime consumers depend on stable protocols and receive a +concrete provider via constructor injection. Concrete providers live +outside this package; ``uipath-core`` does not import them. +""" + +from __future__ import annotations + +from typing import Any, Protocol, runtime_checkable + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from .models import EnforcementMode + +# ---------------------------------------------------------------------- +# Wire-format models +# ---------------------------------------------------------------------- + + +class PolicyContext(BaseModel): + """Caller-supplied selectors for the policy fetch. + + Wrapping the selectors in a model keeps the protocol surface stable + when the server grows new selector dimensions — adding a field here + doesn't change :meth:`GovernancePolicyProvider.get_policy`. + + Today carries only :attr:`is_conversational`; future selectors land + here. + """ + + model_config = ConfigDict(extra="ignore") + + is_conversational: bool | None = None + + +class PolicyResponse(BaseModel): + """Parsed governance backend response. + + Wire envelope:: + + { + "mode": "audit" | "enforce" | "disabled", + "policies": "" + } + + Attributes: + mode: Platform-controlled enforcement mode for the tenant. May + be ``None`` when the backend omits it. A wire value the SDK + doesn't know about parses as ``None`` rather than raising, + so a server-side mode addition can't break agent startup. + policies: Policy pack YAML the caller compiles into its policy + index. May be an empty string when no rules are configured. + """ + + model_config = ConfigDict(extra="ignore") + + mode: EnforcementMode | None = Field(default=None) + policies: str = Field(default="") + + @field_validator("mode", mode="before") + @classmethod + def _coerce_mode(cls, value: object) -> EnforcementMode | None: + if value is None or isinstance(value, EnforcementMode): + return value + try: + return EnforcementMode(value) + except ValueError: + return None + + +class FiredRule(BaseModel): + """Per-rule metadata carried in the ``/runtime/govern`` payload. + + One entry per matching ``guardrail_fallback`` condition. The server + writes one LLMOps trace record per entry, so callers must include + every fired rule even when multiple share the same ``validator``. + """ + + model_config = ConfigDict(populate_by_name=True) + + rule_id: str = Field(alias="ruleId") + rule_name: str = Field(alias="ruleName") + pack_name: str = Field(alias="packName") + validator: str + + +class GovernRequest(BaseModel): + """Request body for the ``/runtime/govern`` compensating governance POST. + + Field aliases match the on-the-wire JSON keys. ``src_timestamp`` is + snake_case on the wire (intentional — preserved verbatim); every + other key is camelCase. + + Job-context fields (``folder_key`` / ``job_key`` / ``process_key`` / + ``reference_id`` / ``agent_version``) are optional; callers omit + them by leaving them ``None``. How unset fields are resolved (e.g. + auto-filled from environment) is the concrete provider's concern, + not part of this wire contract. + + ``trace_id`` is optional. When ``None`` the field is omitted from + the wire JSON (via ``exclude_none=True`` at serialisation). Whether + a concrete provider chooses to populate a missing value before + sending is the provider's concern, not part of this contract. + """ + + model_config = ConfigDict(populate_by_name=True) + + validators: list[str] = Field(alias="type") + rules: list[FiredRule] + data: dict[str, Any] + hook: str + trace_id: str | None = Field(default=None, alias="traceId") + src_timestamp: str # wire key is intentionally snake_case + agent_name: str = Field(alias="agentName") + runtime_id: str = Field(alias="runtimeId") + + folder_key: str | None = Field(default=None, alias="folderKey") + job_key: str | None = Field(default=None, alias="jobKey") + process_key: str | None = Field(default=None, alias="processKey") + reference_id: str | None = Field(default=None, alias="referenceId") + agent_version: str | None = Field(default=None, alias="agentVersion") + + # Runtime identity for governance telemetry; the server stamps these on the + # rule-denied events it emits. Optional — omitted from the wire when None. + agent_framework: str | None = Field(default=None, alias="agentFramework") + agent_type: str | None = Field(default=None, alias="agentType") + runtime_version: str | None = Field(default=None, alias="runtimeVersion") + + +# ---------------------------------------------------------------------- +# Provider protocols +# ---------------------------------------------------------------------- + + +@runtime_checkable +class GovernancePolicyProvider(Protocol): + """Contract for fetching the governance policy pack. + + Implementations expose both a sync and an async fetch. The async + variant is the preferred entry point for hosts running on an event + loop (the host can overlap policy fetch with the rest of agent + setup via ``asyncio.create_task`` and ``await`` the resolved + :class:`PolicyResponse` before constructing the governance + wrapper). The sync variant is kept for callers outside an event + loop (CLI tools, integration tests). + + Any object exposing both ``get_policy(context) -> PolicyResponse`` + and ``async def get_policy_async(context) -> PolicyResponse`` + satisfies this protocol. + """ + + def get_policy(self, context: PolicyContext) -> PolicyResponse: + """Fetch the policy pack for the active org/tenant.""" + ... + + async def get_policy_async(self, context: PolicyContext) -> PolicyResponse: + """Async variant of :meth:`get_policy`. + + Hosts running on an event loop should use this so the fetch + doesn't block the loop and can overlap with other startup + work. + """ + ... + + +@runtime_checkable +class GovernanceCompensationProvider(Protocol): + """Contract for firing the compensating ``/runtime/govern`` POST.""" + + def compensate(self, request: GovernRequest) -> None: + """Fire the compensating governance POST. Fire-and-forget.""" + ... diff --git a/packages/uipath-core/src/uipath/core/guardrails/guardrails.py b/packages/uipath-core/src/uipath/core/guardrails/guardrails.py index fe651c35b..576cc437a 100644 --- a/packages/uipath-core/src/uipath/core/guardrails/guardrails.py +++ b/packages/uipath-core/src/uipath/core/guardrails/guardrails.py @@ -1,7 +1,7 @@ """Guardrails models for UiPath Platform.""" from enum import Enum -from typing import Annotated, Any, Callable, Literal +from typing import Annotated, Any, Callable, Literal, Optional from pydantic import BaseModel, ConfigDict, Field, field_validator @@ -30,6 +30,8 @@ class GuardrailValidationResult(BaseModel): Attributes: result: The validation result type. reason: Textual explanation describing why the validation passed or failed. + span_id: Span ID from the guardrail service response, formatted as a GUID + for trace correlation. None when the response omits the header. """ model_config = ConfigDict(populate_by_name=True) @@ -40,6 +42,11 @@ class GuardrailValidationResult(BaseModel): reason: str = Field( alias="reason", description="Explanation for the validation result." ) + span_id: Optional[str] = Field( + default=None, + alias="spanId", + description="Span ID returned by the guardrail service for trace correlation.", + ) class FieldSource(str, Enum): @@ -227,7 +234,7 @@ class BaseGuardrail(BaseModel): name: str description: str | None = None enabled_for_evals: bool = Field(True, alias="enabledForEvals") - selector: GuardrailSelector + selector: GuardrailSelector | None = None model_config = ConfigDict(populate_by_name=True, extra="allow") diff --git a/packages/uipath-core/src/uipath/core/triggers/__init__.py b/packages/uipath-core/src/uipath/core/triggers/__init__.py index 400462277..b60fc11a2 100644 --- a/packages/uipath-core/src/uipath/core/triggers/__init__.py +++ b/packages/uipath-core/src/uipath/core/triggers/__init__.py @@ -4,11 +4,17 @@ "UiPathResumeTrigger", "UiPathResumeTriggerType", "UiPathApiTrigger", + "UiPathIntegrationTrigger", + "UiPathResumeMetadata", "UiPathResumeTriggerName", + "UIPATH_METADATA_KEY", ] from uipath.core.triggers.trigger import ( + UIPATH_METADATA_KEY, UiPathApiTrigger, + UiPathIntegrationTrigger, + UiPathResumeMetadata, UiPathResumeTrigger, UiPathResumeTriggerName, UiPathResumeTriggerType, diff --git a/packages/uipath-core/src/uipath/core/triggers/trigger.py b/packages/uipath-core/src/uipath/core/triggers/trigger.py index 424245079..508b4ee47 100644 --- a/packages/uipath-core/src/uipath/core/triggers/trigger.py +++ b/packages/uipath-core/src/uipath/core/triggers/trigger.py @@ -1,10 +1,14 @@ """Module defining resume trigger types and data models.""" +from datetime import datetime from enum import Enum from typing import Any from pydantic import BaseModel, ConfigDict, Field +# Reserved key for UiPath-owned metadata embedded in user-visible payloads. +UIPATH_METADATA_KEY = "__uipath" + class UiPathResumeTriggerType(str, Enum): """Constants representing different types of resume job triggers in the system.""" @@ -44,6 +48,19 @@ class UiPathResumeTriggerName(str, Enum): DEEP_RAG_RAW = "DeepRagRaw" +class UiPathResumeMetadata(BaseModel): + """UiPath metadata attached to resume values from multi-trigger interrupts.""" + + trigger_type: UiPathResumeTriggerType | None = Field( + default=None, alias="triggerType" + ) + trigger_name: UiPathResumeTriggerName | None = Field( + default=None, alias="triggerName" + ) + + model_config = ConfigDict(validate_by_name=True) + + class UiPathApiTrigger(BaseModel): """API resume trigger request.""" @@ -53,6 +70,25 @@ class UiPathApiTrigger(BaseModel): model_config = ConfigDict(validate_by_name=True) +class UiPathIntegrationTrigger(BaseModel): + """Integration Services (Inbox) resume trigger request. + + Mirrors Orchestrator's `IntegrationResumeDto`: the configuration needed to + register a remote event trigger through the Connections service and + correlate the eventual payload back to the suspended job via `inbox_id`. + """ + + connector: str = Field(alias="connector") + connection_id: str = Field(alias="connectionId") + operation: str = Field(alias="operation") + object_name: str = Field(alias="objectName") + filter_expression: str | None = Field(default=None, alias="filterExpression") + parameters: dict[str, str] | None = Field(default=None, alias="parameters") + inbox_id: str = Field(alias="inboxId") + + model_config = ConfigDict(validate_by_name=True) + + class UiPathResumeTrigger(BaseModel): """Information needed to resume execution.""" @@ -65,6 +101,10 @@ class UiPathResumeTrigger(BaseModel): ) item_key: str | None = Field(default=None, alias="itemKey") api_resume: UiPathApiTrigger | None = Field(default=None, alias="apiResume") + integration_resume: UiPathIntegrationTrigger | None = Field( + default=None, alias="integrationResume" + ) + resume_time: datetime | None = Field(default=None, alias="resumeTime") folder_path: str | None = Field(default=None, alias="folderPath") folder_key: str | None = Field(default=None, alias="folderKey") payload: Any | None = Field(default=None, alias="interruptObject", exclude=True) diff --git a/packages/uipath-core/src/uipath/core/workspace/__init__.py b/packages/uipath-core/src/uipath/core/workspace/__init__.py new file mode 100644 index 000000000..c05992aae --- /dev/null +++ b/packages/uipath-core/src/uipath/core/workspace/__init__.py @@ -0,0 +1,8 @@ +"""UiPath workspace hydration shared contracts.""" + +from .protocols import AttachmentsProtocol, JobsProtocol + +__all__ = [ + "AttachmentsProtocol", + "JobsProtocol", +] diff --git a/packages/uipath-core/src/uipath/core/workspace/protocols.py b/packages/uipath-core/src/uipath/core/workspace/protocols.py new file mode 100644 index 000000000..c49c7bc59 --- /dev/null +++ b/packages/uipath-core/src/uipath/core/workspace/protocols.py @@ -0,0 +1,78 @@ +"""Service protocols for workspace hydration backend interactions.""" + +from __future__ import annotations + +from typing import Protocol, overload, runtime_checkable +from uuid import UUID + + +@runtime_checkable +class AttachmentsProtocol(Protocol): + """Subset of the UiPath attachments service used by workspace hydration.""" + + async def download_async( + self, + *, + key: UUID, + destination_path: str, + folder_key: str | None = None, + folder_path: str | None = None, + ) -> str: + """Download an attachment to a local path.""" + + # Overloads mirror the concrete AttachmentsService: content XOR source_path, + # so the real service is a structural subtype of this protocol. + @overload + async def upload_async( + self, + *, + name: str, + content: str | bytes, + folder_key: str | None = None, + folder_path: str | None = None, + ) -> UUID: ... + + @overload + async def upload_async( + self, + *, + name: str, + source_path: str, + folder_key: str | None = None, + folder_path: str | None = None, + ) -> UUID: ... + + async def upload_async( + self, + *, + name: str, + content: str | bytes | None = None, + source_path: str | None = None, + folder_key: str | None = None, + folder_path: str | None = None, + ) -> UUID: + """Upload content or a local file and return its attachment key.""" + + +@runtime_checkable +class JobsProtocol(Protocol): + """Subset of the UiPath jobs service used by workspace hydration.""" + + async def list_attachments_async( + self, + *, + job_key: UUID, + folder_key: str | None = None, + folder_path: str | None = None, + ) -> list[str]: + """List the attachment ids linked to a job.""" + + async def link_attachment_async( + self, + *, + job_key: UUID, + attachment_key: UUID, + folder_key: str | None = None, + folder_path: str | None = None, + ) -> None: + """Link an existing attachment to a job.""" diff --git a/packages/uipath-core/tests/adapters/__init__.py b/packages/uipath-core/tests/adapters/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/uipath-core/tests/adapters/test_evaluator.py b/packages/uipath-core/tests/adapters/test_evaluator.py new file mode 100644 index 000000000..83aa70b5e --- /dev/null +++ b/packages/uipath-core/tests/adapters/test_evaluator.py @@ -0,0 +1,104 @@ +"""Tests for EvaluatorProtocol. + +The protocol is a structural type. These tests verify two things: + +1. A class whose method shapes match the protocol passes ``isinstance`` + against the ``runtime_checkable`` Protocol. +2. Subclassing the Protocol and calling ``super().`` actually + executes the stub bodies — this both documents that the stubs are + safely callable (they return ``None``) and brings the contract module + to full line coverage. +""" + +from __future__ import annotations + +from typing import Any + +from uipath.core.adapters import EvaluatorProtocol + + +class _MissingMethodEvaluator: + """Only implements one method — fails the structural check.""" + + def evaluate_before_agent(self, *args: Any, **kwargs: Any) -> Any: + return None + + +class _CompleteEvaluator: + """All six methods present with the expected names — passes ``isinstance``.""" + + def evaluate_before_agent(self, *args: Any, **kwargs: Any) -> Any: + return "before-agent" + + def evaluate_after_agent(self, *args: Any, **kwargs: Any) -> Any: + return "after-agent" + + def evaluate_before_model(self, *args: Any, **kwargs: Any) -> Any: + return "before-model" + + def evaluate_after_model(self, *args: Any, **kwargs: Any) -> Any: + return "after-model" + + def evaluate_tool_call(self, *args: Any, **kwargs: Any) -> Any: + return "tool-call" + + def evaluate_after_tool(self, *args: Any, **kwargs: Any) -> Any: + return "after-tool" + + +class _ProtocolSubclass(EvaluatorProtocol): + """Subclass that delegates to ``super()`` — exercises the stub bodies. + + Each override calls ``super().(...)`` so the ``...`` body of + the Protocol method actually executes (returns ``None``). + """ + + def evaluate_before_agent(self, *args: Any, **kwargs: Any) -> Any: + return super().evaluate_before_agent(*args, **kwargs) # type: ignore[safe-super] + + def evaluate_after_agent(self, *args: Any, **kwargs: Any) -> Any: + return super().evaluate_after_agent(*args, **kwargs) # type: ignore[safe-super] + + def evaluate_before_model(self, *args: Any, **kwargs: Any) -> Any: + return super().evaluate_before_model(*args, **kwargs) # type: ignore[safe-super] + + def evaluate_after_model(self, *args: Any, **kwargs: Any) -> Any: + return super().evaluate_after_model(*args, **kwargs) # type: ignore[safe-super] + + def evaluate_tool_call(self, *args: Any, **kwargs: Any) -> Any: + return super().evaluate_tool_call(*args, **kwargs) # type: ignore[safe-super] + + def evaluate_after_tool(self, *args: Any, **kwargs: Any) -> Any: + return super().evaluate_after_tool(*args, **kwargs) # type: ignore[safe-super] + + +# --------------------------------------------------------------------------- +# Structural conformance +# --------------------------------------------------------------------------- + + +def test_complete_evaluator_is_recognized_by_runtime_check(): + """A class with all six methods passes ``isinstance`` against the protocol.""" + assert isinstance(_CompleteEvaluator(), EvaluatorProtocol) + + +def test_partial_evaluator_is_rejected_by_runtime_check(): + """A class missing methods does NOT pass the structural check.""" + assert not isinstance(_MissingMethodEvaluator(), EvaluatorProtocol) + + +# --------------------------------------------------------------------------- +# Stub-body execution (line coverage for the ``...`` placeholders) +# --------------------------------------------------------------------------- + + +def test_protocol_subclass_methods_execute_stub_bodies(): + """Calling each method via ``super()`` executes the stub body and returns None.""" + e = _ProtocolSubclass() + + assert e.evaluate_before_agent("input", "agent", "rt") is None + assert e.evaluate_after_agent("output", "agent", "rt") is None + assert e.evaluate_before_model("input", "agent", "rt") is None + assert e.evaluate_after_model("output", "agent", "rt") is None + assert e.evaluate_tool_call("tool", {"arg": 1}, "agent", "rt") is None + assert e.evaluate_after_tool("tool", "result", "agent", "rt") is None diff --git a/packages/uipath-core/tests/chat/__init__.py b/packages/uipath-core/tests/chat/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/uipath-core/tests/chat/test_message.py b/packages/uipath-core/tests/chat/test_message.py new file mode 100644 index 000000000..f56178fc5 --- /dev/null +++ b/packages/uipath-core/tests/chat/test_message.py @@ -0,0 +1,87 @@ +"""Tests for `UiPathConversationMessage` input validation. + +Conversational Agent Service contract treats `role` + `contentParts` as +the load-bearing fields for an inbound user message. `messageId` and +`contentPartId` are GUIDs that identify entities in the conversation +hierarchy; when omitted on input, the model fills them with fresh +UUIDs (matching what `uipath dev` does server-side). `createdAt`, +`updatedAt`, `spanId`, and `toolCalls` are server-allocated and absent +from client input. + +These tests pin that behavior so `--input-file` payloads from +`uip codedagent run` validate against the model without requiring +callers to hand-generate UUIDs. +""" + +from __future__ import annotations + +from uipath.core.chat import UiPathConversationMessage + + +def test_minimal_user_message_validates_and_fills_ids() -> None: + msg = UiPathConversationMessage.model_validate( + { + "role": "user", + "contentParts": [ + { + "mimeType": "text/plain", + "data": {"inline": "hello world"}, + } + ], + } + ) + assert msg.role == "user" + assert msg.tool_calls == [] + assert msg.created_at is None + assert msg.updated_at is None + assert msg.message_id # auto-generated UUID + assert msg.content_parts[0].content_part_id # auto-generated UUID + assert msg.content_parts[0].citations == [] + + +def test_explicit_ids_are_preserved() -> None: + msg = UiPathConversationMessage.model_validate( + { + "messageId": "00000000-0000-0000-0000-000000000001", + "role": "user", + "contentParts": [ + { + "contentPartId": "00000000-0000-0000-0000-000000000002", + "mimeType": "text/plain", + "data": {"inline": "hello world"}, + } + ], + } + ) + assert msg.message_id == "00000000-0000-0000-0000-000000000001" + assert ( + msg.content_parts[0].content_part_id == "00000000-0000-0000-0000-000000000002" + ) + + +def test_content_part_metadata_is_preserved() -> None: + msg = UiPathConversationMessage.model_validate( + { + "role": "assistant", + "contentParts": [ + { + "mimeType": "text/markdown", + "name": "plan.md", + "data": { + "uri": "urn:uipath:cas:file:orchestrator:" + "00000000-0000-0000-0000-000000000003" + }, + "metaData": {"fileKind": "workspace", "sha256": "abc123"}, + } + ], + } + ) + + assert msg.content_parts[0].metadata == { + "fileKind": "workspace", + "sha256": "abc123", + } + assert msg.model_dump(by_alias=True)["contentParts"][0]["metaData"] == { + "fileKind": "workspace", + "sha256": "abc123", + } diff --git a/packages/uipath-core/tests/governance/__init__.py b/packages/uipath-core/tests/governance/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/uipath-core/tests/governance/test_config.py b/packages/uipath-core/tests/governance/test_config.py new file mode 100644 index 000000000..54642a413 --- /dev/null +++ b/packages/uipath-core/tests/governance/test_config.py @@ -0,0 +1,54 @@ +"""Tests for the governance feature-flag gate.""" + +from __future__ import annotations + +import pytest + +from uipath.core.feature_flags import FeatureFlags +from uipath.core.governance.config import ( + GOVERNANCE_FEATURE_FLAG, + is_governance_enabled, +) + + +@pytest.fixture(autouse=True) +def _reset_flags(): + """Each test starts and ends with a clean flags registry.""" + FeatureFlags.reset_flags() + yield + FeatureFlags.reset_flags() + + +def test_governance_flag_name_is_stable(): + """The flag name is a public contract shared with the runtime layer.""" + assert GOVERNANCE_FEATURE_FLAG == "EnablePythonGovernanceChecker" + + +def test_is_governance_enabled_defaults_to_false(): + """With nothing configured, the gate defaults to disabled. + + The platform / host runtime must explicitly opt into governance + (programmatically via :class:`FeatureFlags`, via gitops, or via the + ``UIPATH_FEATURE_EnablePythonGovernanceChecker`` env var). This + keeps the SDK safe-by-default for callers that haven't yet + integrated with the governance backend. + """ + assert is_governance_enabled() is False + + +def test_is_governance_enabled_respects_programmatic_disable(): + """Programmatic ``False`` flips the gate off.""" + FeatureFlags.configure_flags({GOVERNANCE_FEATURE_FLAG: False}) + assert is_governance_enabled() is False + + +def test_is_governance_enabled_respects_programmatic_enable(): + """Programmatic ``True`` keeps the gate on.""" + FeatureFlags.configure_flags({GOVERNANCE_FEATURE_FLAG: True}) + assert is_governance_enabled() is True + + +def test_is_governance_enabled_reads_env_var_fallback(monkeypatch): + """When nothing is configured programmatically, the env-var fallback wins.""" + monkeypatch.setenv(f"UIPATH_FEATURE_{GOVERNANCE_FEATURE_FLAG}", "false") + assert is_governance_enabled() is False diff --git a/packages/uipath-core/tests/governance/test_exceptions.py b/packages/uipath-core/tests/governance/test_exceptions.py new file mode 100644 index 000000000..5e8738a15 --- /dev/null +++ b/packages/uipath-core/tests/governance/test_exceptions.py @@ -0,0 +1,204 @@ +"""Tests for GovernanceBlockException constructors. + +The classmethod constructors (:meth:`from_violation`, +:meth:`from_audit_record`) form the documented contract that the +evaluator and adapter packages depend on — the evaluator only ever +builds a block via ``from_audit_record``. These tests pin the message +format and attribute population so a future refactor cannot silently +drop the rule id, name, or detail. +""" + +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest + +from uipath.core.governance.exceptions import ( + GovernanceBlockException, + GovernanceViolation, + Severity, +) +from uipath.core.governance.models import ( + Action, + AuditRecord, + LifecycleHook, + RuleEvaluation, +) + +# --------------------------------------------------------------------------- +# GovernanceViolation +# --------------------------------------------------------------------------- + + +def test_violation_defaults_to_high_severity(): + v = GovernanceViolation(rule_id="A-1", rule_name="No PII", detail="ssn leaked") + assert v.severity == Severity.HIGH + + +def test_violation_severity_can_be_overridden(): + v = GovernanceViolation( + rule_id="A-1", + rule_name="No PII", + detail="ssn leaked", + severity=Severity.CRITICAL, + ) + assert v.severity == Severity.CRITICAL + + +# --------------------------------------------------------------------------- +# GovernanceBlockException base constructor +# --------------------------------------------------------------------------- + + +def test_default_constructor_emits_prefixed_message(): + exc = GovernanceBlockException() + assert "[Governance Policy Violation]" in str(exc) + assert exc.violation is None + assert exc.audit_record is None + + +def test_default_constructor_carries_default_rule_metadata(): + """Constructing without context still gives the documented fallback IDs.""" + exc = GovernanceBlockException() + assert exc.rule_id == "POLICY" + assert exc.rule_name == "Governance Policy" + + +def test_explicit_message_is_used_verbatim(): + exc = GovernanceBlockException("custom message") + assert str(exc) == "custom message" + + +def test_error_code_constant_for_orchestrator_categorization(): + """error_code is a class-level constant the Orchestrator UI reads.""" + assert GovernanceBlockException.error_code == "GOVERNANCE_POLICY_VIOLATION" + exc = GovernanceBlockException() + assert exc.error_code == "GOVERNANCE_POLICY_VIOLATION" + + +# --------------------------------------------------------------------------- +# from_violation +# --------------------------------------------------------------------------- + + +def test_from_violation_populates_rule_metadata(): + v = GovernanceViolation(rule_id="A-1", rule_name="No PII", detail="ssn leaked") + exc = GovernanceBlockException.from_violation(v) + assert exc.rule_id == "A-1" + assert exc.rule_name == "No PII" + assert exc.violation is v + + +def test_from_violation_message_includes_rule_id_name_detail(): + v = GovernanceViolation(rule_id="A-1", rule_name="No PII", detail="ssn leaked") + msg = str(GovernanceBlockException.from_violation(v)) + assert "A-1" in msg + assert "No PII" in msg + assert "ssn leaked" in msg + assert "[Governance Policy Violation]" in msg + + +# --------------------------------------------------------------------------- +# from_audit_record +# --------------------------------------------------------------------------- + + +def _audit_record_with(*evaluations: RuleEvaluation) -> AuditRecord: + return AuditRecord( + timestamp=datetime.now(timezone.utc), + agent_name="agent", + runtime_id="run-1", + hook=LifecycleHook.BEFORE_AGENT, + evaluations=list(evaluations), + final_action=Action.DENY, + ) + + +def test_from_audit_record_picks_first_matched_rule(): + """Even when later evaluations matched, the first matched wins the message.""" + audit = _audit_record_with( + RuleEvaluation( + rule_id="UNMATCHED", + rule_name="Did not fire", + matched=False, + detail="", + action=Action.ALLOW, + ), + RuleEvaluation( + rule_id="MATCHED-FIRST", + rule_name="First match", + matched=True, + detail="bad input", + action=Action.DENY, + ), + RuleEvaluation( + rule_id="MATCHED-SECOND", + rule_name="Second match", + matched=True, + detail="also bad", + action=Action.DENY, + ), + ) + + exc = GovernanceBlockException.from_audit_record(audit) + assert exc.rule_id == "MATCHED-FIRST" + assert exc.rule_name == "First match" + assert "bad input" in str(exc) + assert exc.audit_record is audit + + +def test_from_audit_record_falls_back_when_no_match(): + """When the audit has no matches, the exception is still constructible.""" + audit = _audit_record_with( + RuleEvaluation( + rule_id="UNMATCHED", + rule_name="Did not fire", + matched=False, + detail="", + action=Action.ALLOW, + ) + ) + + exc = GovernanceBlockException.from_audit_record(audit) + assert "Rules evaluated: 1" in str(exc) + assert exc.audit_record is audit + + +def test_from_audit_record_matched_detail_default_when_empty(): + """A matched evaluation with empty detail still produces a sensible message.""" + audit = _audit_record_with( + RuleEvaluation( + rule_id="A-1", + rule_name="No PII", + matched=True, + detail="", # empty + action=Action.DENY, + ) + ) + + msg = str(GovernanceBlockException.from_audit_record(audit)) + assert "A-1" in msg + assert "No PII" in msg + # Falls back to a non-empty detail string. + assert "Policy violation detected" in msg + + +# --------------------------------------------------------------------------- +# Exception identity — must be a real Exception so callers can catch broadly +# --------------------------------------------------------------------------- + + +def test_block_exception_is_exception_subclass(): + assert issubclass(GovernanceBlockException, Exception) + + +def test_block_exception_can_be_caught_via_base_exception(): + try: + raise GovernanceBlockException.from_violation( + GovernanceViolation(rule_id="A-1", rule_name="X", detail="d") + ) + except Exception as e: # noqa: BLE001 - intentional broad catch + assert isinstance(e, GovernanceBlockException) + else: + pytest.fail("Did not raise") diff --git a/packages/uipath-core/tests/governance/test_providers.py b/packages/uipath-core/tests/governance/test_providers.py new file mode 100644 index 000000000..21e5f1703 --- /dev/null +++ b/packages/uipath-core/tests/governance/test_providers.py @@ -0,0 +1,172 @@ +"""Tests for the governance provider protocols + wire-format models.""" + +from __future__ import annotations + +import pytest + +from uipath.core.governance import ( + EnforcementMode, + FiredRule, + GovernanceCompensationProvider, + GovernancePolicyProvider, + GovernRequest, + PolicyContext, + PolicyResponse, +) + + +class _FakePolicyProvider: + def __init__(self) -> None: + self.calls: list[PolicyContext] = [] + self.async_calls: list[PolicyContext] = [] + + def get_policy(self, context: PolicyContext) -> PolicyResponse: + self.calls.append(context) + return PolicyResponse(mode=EnforcementMode.ENFORCE, policies="rules: []") + + async def get_policy_async(self, context: PolicyContext) -> PolicyResponse: + self.async_calls.append(context) + return PolicyResponse(mode=EnforcementMode.ENFORCE, policies="rules: []") + + +class _FakeCompensationProvider: + def __init__(self) -> None: + self.calls: list[GovernRequest] = [] + + def compensate(self, request: GovernRequest) -> None: + self.calls.append(request) + + +def _make_request() -> GovernRequest: + return GovernRequest( + validators=["pii_detection"], + rules=[ + FiredRule( + rule_id="ASI-01", + rule_name="Block PII in flight", + pack_name="agent-safety", + validator="pii_detection", + ) + ], + data={"prompt": "hi"}, + hook="before_model", + trace_id="0123456789abcdef0123456789abcdef", + src_timestamp="2026-06-22T10:00:00Z", + agent_name="my-agent", + runtime_id="runtime-1", + ) + + +class TestPolicyContext: + def test_defaults(self) -> None: + ctx = PolicyContext() + assert ctx.is_conversational is None + + def test_ignores_unknown_fields(self) -> None: + ctx = PolicyContext.model_validate( + {"is_conversational": True, "future_selector": "x"} + ) + assert ctx.is_conversational is True + + +class TestPolicyResponse: + def test_defaults(self) -> None: + response = PolicyResponse() + assert response.mode is None + assert response.policies == "" + + @pytest.mark.parametrize( + ("wire_value", "expected"), + [ + ("audit", EnforcementMode.AUDIT), + ("enforce", EnforcementMode.ENFORCE), + ("disabled", EnforcementMode.DISABLED), + ], + ) + def test_parses_known_modes( + self, wire_value: str, expected: EnforcementMode + ) -> None: + response = PolicyResponse.model_validate({"mode": wire_value}) + assert response.mode is expected + + def test_unknown_mode_falls_back_to_none(self) -> None: + # Forward-compat: a server-added mode the SDK doesn't know about + # must not break agent startup. Parses as None so the runtime + # falls back to its safe default rather than raising. + response = PolicyResponse.model_validate({"mode": "ludicrous"}) + assert response.mode is None + + +class TestGovernRequest: + def test_serializes_wire_aliases(self) -> None: + payload = _make_request().model_dump(by_alias=True, exclude_none=True) + assert payload["type"] == ["pii_detection"] + assert payload["traceId"] == "0123456789abcdef0123456789abcdef" + assert payload["agentName"] == "my-agent" + assert payload["runtimeId"] == "runtime-1" + # src_timestamp is intentionally snake_case on the wire. + assert payload["src_timestamp"] == "2026-06-22T10:00:00Z" + # Optional job-context fields left None → excluded. + for absent in ( + "folderKey", + "jobKey", + "processKey", + "referenceId", + "agentVersion", + ): + assert absent not in payload + + +class TestProtocolConformance: + """`runtime_checkable` Protocols should accept structurally-matching objects.""" + + def test_fake_policy_provider_satisfies_protocol(self) -> None: + provider = _FakePolicyProvider() + assert isinstance(provider, GovernancePolicyProvider) + + def test_fake_compensation_provider_satisfies_protocol(self) -> None: + provider = _FakeCompensationProvider() + assert isinstance(provider, GovernanceCompensationProvider) + + def test_object_without_methods_rejected(self) -> None: + class _NotAProvider: + pass + + assert not isinstance(_NotAProvider(), GovernancePolicyProvider) + assert not isinstance(_NotAProvider(), GovernanceCompensationProvider) + + +class TestEndToEndDispatch: + """Caller passes a provider directly to the consumer (no global registry).""" + + def test_policy_round_trip(self) -> None: + provider = _FakePolicyProvider() + response = provider.get_policy(PolicyContext(is_conversational=True)) + + assert response.mode is EnforcementMode.ENFORCE + assert provider.calls == [PolicyContext(is_conversational=True)] + + @pytest.mark.asyncio + async def test_policy_round_trip_async(self) -> None: + """The async variant is the preferred entry point for event-loop hosts. + + Hosts running ``await provider.get_policy_async(ctx)`` overlap + the fetch with the rest of agent setup; the sync ``get_policy`` + path remains for callers outside an event loop. + """ + provider = _FakePolicyProvider() + response = await provider.get_policy_async( + PolicyContext(is_conversational=False) + ) + + assert response.mode is EnforcementMode.ENFORCE + assert provider.async_calls == [PolicyContext(is_conversational=False)] + # Sync slot stays untouched — the two entrypoints are independent. + assert provider.calls == [] + + def test_compensation_round_trip(self) -> None: + provider = _FakeCompensationProvider() + request = _make_request() + provider.compensate(request) + + assert provider.calls == [request] diff --git a/packages/uipath-core/tests/triggers/test_resume_metadata.py b/packages/uipath-core/tests/triggers/test_resume_metadata.py new file mode 100644 index 000000000..99418fd1b --- /dev/null +++ b/packages/uipath-core/tests/triggers/test_resume_metadata.py @@ -0,0 +1,27 @@ +from uipath.core.triggers import ( + UiPathResumeMetadata, + UiPathResumeTriggerName, + UiPathResumeTriggerType, +) + + +def test_resume_metadata_accepts_trigger_aliases() -> None: + metadata = UiPathResumeMetadata.model_validate( + { + "triggerType": "Timer", + "triggerName": "Timer", + } + ) + + assert metadata.trigger_type == UiPathResumeTriggerType.TIMER + assert metadata.trigger_name == UiPathResumeTriggerName.TIMER + + +def test_resume_metadata_accepts_field_names() -> None: + metadata = UiPathResumeMetadata( + trigger_type=UiPathResumeTriggerType.API, + trigger_name=UiPathResumeTriggerName.API, + ) + + assert metadata.trigger_type == UiPathResumeTriggerType.API + assert metadata.trigger_name == UiPathResumeTriggerName.API diff --git a/packages/uipath-core/tests/workspace/__init__.py b/packages/uipath-core/tests/workspace/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/uipath-core/tests/workspace/test_protocols.py b/packages/uipath-core/tests/workspace/test_protocols.py new file mode 100644 index 000000000..0c9cd11c4 --- /dev/null +++ b/packages/uipath-core/tests/workspace/test_protocols.py @@ -0,0 +1,68 @@ +"""Structural-conformance tests for the workspace hydration protocols.""" + +from __future__ import annotations + +from uuid import UUID, uuid4 + +from uipath.core.workspace import AttachmentsProtocol, JobsProtocol + + +class _FakeAttachments: + async def download_async( + self, + *, + key: UUID, + destination_path: str, + folder_key: str | None = None, + folder_path: str | None = None, + ) -> str: + return destination_path + + async def upload_async( + self, + *, + name: str, + content: str | bytes | None = None, + source_path: str | None = None, + folder_key: str | None = None, + folder_path: str | None = None, + ) -> UUID: + return uuid4() + + +class _FakeJobs: + async def list_attachments_async( + self, + *, + job_key: UUID, + folder_key: str | None = None, + folder_path: str | None = None, + ) -> list[str]: + return [] + + async def link_attachment_async( + self, + *, + job_key: UUID, + attachment_key: UUID, + folder_key: str | None = None, + folder_path: str | None = None, + ) -> None: + return None + + +class _NotAService: + pass + + +def test_attachments_service_satisfies_protocol() -> None: + assert isinstance(_FakeAttachments(), AttachmentsProtocol) + + +def test_jobs_service_satisfies_protocol() -> None: + assert isinstance(_FakeJobs(), JobsProtocol) + + +def test_unrelated_object_does_not_satisfy_protocols() -> None: + assert not isinstance(_NotAService(), AttachmentsProtocol) + assert not isinstance(_NotAService(), JobsProtocol) diff --git a/packages/uipath-core/uv.lock b/packages/uipath-core/uv.lock index 2544216df..96df7523f 100644 --- a/packages/uipath-core/uv.lock +++ b/packages/uipath-core/uv.lock @@ -2,6 +2,10 @@ version = 1 revision = 3 requires-python = ">=3.11" +[options] +exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. +exclude-newer-span = "P2D" + [[package]] name = "annotated-types" version = "0.7.0" @@ -1007,7 +1011,7 @@ wheels = [ [[package]] name = "uipath-core" -version = "0.5.10" +version = "0.5.31" source = { editable = "." } dependencies = [ { name = "opentelemetry-instrumentation" }, diff --git a/packages/uipath-platform/pyproject.toml b/packages/uipath-platform/pyproject.toml index ba5634ef1..eab210d34 100644 --- a/packages/uipath-platform/pyproject.toml +++ b/packages/uipath-platform/pyproject.toml @@ -1,14 +1,15 @@ [project] name = "uipath-platform" -version = "0.1.18" +version = "0.2.17" description = "HTTP client library for programmatic access to UiPath Platform" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" dependencies = [ + "anyio>=4.0.0", "httpx>=0.28.1", "tenacity>=9.0.0", "truststore>=0.10.1", - "uipath-core>=0.5.8, <0.6.0", + "uipath-core>=0.5.30, <0.6.0", "pydantic-function-models>=0.1.11", "sqlparse>=0.5.5", ] @@ -98,15 +99,39 @@ warn_required_dynamic_aliases = true [tool.pytest.ini_options] testpaths = ["tests"] python_files = "test_*.py" -addopts = "-ra -q --cov=src/uipath --cov-report=term-missing" +addopts = "-ra -q --cov=src --cov-report=term-missing -m 'not e2e'" asyncio_default_fixture_loop_scope = "function" asyncio_mode = "auto" +markers = [ + "e2e: end-to-end tests against real ECS/LLMOps (requires UIPATH_URL, UIPATH_ACCESS_TOKEN, UIPATH_FOLDER_KEY)", +] + +[tool.coverage.run] +source = ["src"] +relative_files = true +omit = [ + "*/tests/*", + "*/__pycache__/*", + "*/site-packages/*", + "*/conftest.py", +] [tool.coverage.report] show_missing = true +precision = 2 +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "raise NotImplementedError", + "if TYPE_CHECKING:", + "@(abc\\.)?abstractmethod", +] -[tool.coverage.run] -source = ["src"] +[tool.uv] +exclude-newer = "2 days" + +[tool.uv.exclude-newer-package] +uipath-core = false [tool.uv.sources] uipath-core = { path = "../uipath-core", editable = true } diff --git a/packages/uipath-platform/src/uipath/platform/__init__.py b/packages/uipath-platform/src/uipath/platform/__init__.py index 61abb103c..e9535b9f2 100644 --- a/packages/uipath-platform/src/uipath/platform/__init__.py +++ b/packages/uipath-platform/src/uipath/platform/__init__.py @@ -34,7 +34,33 @@ ``` """ -from ._uipath import UiPath -from .common import UiPathApiConfig, UiPathExecutionContext +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ._uipath import UiPath + from .common import UiPathApiConfig, UiPathExecutionContext __all__ = ["UiPathApiConfig", "UiPath", "UiPathExecutionContext"] + + +def __getattr__(name: str): + """Resolve top-level exports on demand. + + Keeps this package's ``__init__`` cheap so lightweight submodules such as + ``uipath.platform.constants`` can be imported without pulling in the + ``UiPath`` facade and the full service layer. The heavy import happens only + when ``UiPath`` (or a config type) is actually accessed. + """ + if name == "UiPath": + from ._uipath import UiPath + + return UiPath + if name in ("UiPathApiConfig", "UiPathExecutionContext"): + from . import common + + return getattr(common, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__() -> list[str]: + return sorted(__all__) diff --git a/packages/uipath-platform/src/uipath/platform/_uipath.py b/packages/uipath-platform/src/uipath/platform/_uipath.py index 87c3a17f0..98af7b8b6 100644 --- a/packages/uipath-platform/src/uipath/platform/_uipath.py +++ b/packages/uipath-platform/src/uipath/platform/_uipath.py @@ -8,10 +8,10 @@ from .action_center import TasksService from .agenthub._agenthub_service import AgentHubService from .agenthub._remote_a2a_service import RemoteA2aService +from .automation_ops import AutomationOpsService from .chat import ConversationsService, UiPathLlmChatService, UiPathOpenAIService from .common import ( ApiClient, - ExternalApplicationService, UiPathApiConfig, UiPathExecutionContext, ) @@ -21,7 +21,10 @@ from .documents import DocumentsService from .entities import EntitiesService from .errors import BaseUrlMissingError, SecretMissingError +from .external_applications import ExternalApplicationService +from .governance import GovernanceService from .guardrails import GuardrailsService +from .memory import MemoryService from .orchestrator import ( AssetsService, AttachmentsService, @@ -33,7 +36,9 @@ ProcessesService, QueuesService, ) +from .pii_detection import PiiDetectionService from .resource_catalog import ResourceCatalogService +from .semantic_proxy import SemanticProxyService def _has_valid_client_credentials( @@ -113,6 +118,10 @@ def context_grounding(self) -> ContextGroundingService: self.buckets, ) + @property + def memory(self) -> MemoryService: + return MemoryService(self._config, self._execution_context, self.folders) + @property def documents(self) -> DocumentsService: return DocumentsService(self._config, self._execution_context) @@ -139,7 +148,9 @@ def llm(self) -> UiPathLlmChatService: @property def entities(self) -> EntitiesService: - return EntitiesService(self._config, self._execution_context) + return EntitiesService( + self._config, self._execution_context, folders_service=self.folders + ) @cached_property def resource_catalog(self) -> ResourceCatalogService: @@ -159,6 +170,10 @@ def mcp(self) -> McpService: def guardrails(self) -> GuardrailsService: return GuardrailsService(self._config, self._execution_context) + @cached_property + def governance(self) -> GovernanceService: + return GovernanceService(self._config, self._execution_context) + @property def agenthub(self) -> AgentHubService: return AgentHubService(self._config, self._execution_context, self.folders) @@ -171,6 +186,18 @@ def remote_a2a(self) -> RemoteA2aService: def orchestrator_setup(self) -> OrchestratorSetupService: return OrchestratorSetupService(self._config, self._execution_context) + @property + def automation_ops(self) -> AutomationOpsService: + return AutomationOpsService(self._config, self._execution_context) + + @property + def pii_detection(self) -> PiiDetectionService: + return PiiDetectionService(self._config, self._execution_context) + + @property + def semantic_proxy(self) -> SemanticProxyService: + return SemanticProxyService(self._config, self._execution_context) + @property def automation_tracker(self) -> AutomationTrackerService: return AutomationTrackerService(self._config, self._execution_context) diff --git a/packages/uipath-platform/src/uipath/platform/action_center/_tasks_service.py b/packages/uipath-platform/src/uipath/platform/action_center/_tasks_service.py index 662109ce4..dea78f882 100644 --- a/packages/uipath-platform/src/uipath/platform/action_center/_tasks_service.py +++ b/packages/uipath-platform/src/uipath/platform/action_center/_tasks_service.py @@ -5,16 +5,17 @@ from uipath.core.tracing import traced +from uipath.platform.constants import ( + ENV_TENANT_ID, + HEADER_TENANT_ID, +) + from ..common._base_service import BaseService from ..common._bindings import resource_override from ..common._config import UiPathApiConfig, UiPathConfig from ..common._execution_context import UiPathExecutionContext from ..common._folder_context import FolderContext, header_folder from ..common._models import Endpoint, RequestSpec -from ..common.constants import ( - ENV_TENANT_ID, - HEADER_TENANT_ID, -) from .task_schema import TaskSchema from .tasks import Task, TaskRecipient, TaskRecipientType @@ -118,10 +119,34 @@ def _create_spec( ), } + _apply_priority_labels_and_actionable_toggle( + json_payload, priority, labels, is_actionable_message_enabled + ) + _apply_task_source(json_payload, source_name) + + return RequestSpec( + method="POST", + endpoint=Endpoint("/orchestrator_/tasks/AppTasks/CreateAppTask"), + json=json_payload, + headers=header_folder(app_folder_key, app_folder_path), + ) + + +def _apply_priority_labels_and_actionable_toggle( + payload: Dict[str, Any], + priority: Optional[str], + labels: Optional[List[str]], + is_actionable_message_enabled: Optional[bool], +) -> None: + """Apply priority / tags / isActionableMessageEnabled to ``payload`` in-place. + + Shared between AppTask and QuickForm spec builders — they handle these three + optional fields identically. + """ if priority and (normalized_priority := _normalize_priority(priority)): - json_payload["priority"] = normalized_priority + payload["priority"] = normalized_priority if labels is not None: - json_payload["tags"] = [ + payload["tags"] = [ { "name": label, "displayName": label, @@ -131,37 +156,29 @@ def _create_spec( for label in labels ] if is_actionable_message_enabled is not None: - json_payload["isActionableMessageEnabled"] = is_actionable_message_enabled + payload["isActionableMessageEnabled"] = is_actionable_message_enabled - project_id = UiPathConfig.project_id - trace_id = UiPathConfig.trace_id - if project_id and trace_id: - folder_key = UiPathConfig.folder_key - job_key = UiPathConfig.job_key - process_key = UiPathConfig.process_uuid +def _apply_task_source(payload: Dict[str, Any], source_name: str) -> None: + """Populate ``payload["taskSource"]`` when UiPathConfig has project_id + trace_id. - task_source_metadata: Dict[str, Any] = { + Shared between AppTask and QuickForm spec builders — the taskSource block is + identical for both task types. + """ + project_id = UiPathConfig.project_id + trace_id = UiPathConfig.trace_id + if not (project_id and trace_id): + return + payload["taskSource"] = { + "sourceName": source_name, + "sourceId": project_id, + "taskSourceMetadata": { "InstanceId": trace_id, - "FolderKey": folder_key, - "JobKey": job_key, - "ProcessKey": process_key, - } - - task_source = { - "sourceName": source_name, - "sourceId": project_id, - "taskSourceMetadata": task_source_metadata, - } - - json_payload["taskSource"] = task_source - - return RequestSpec( - method="POST", - endpoint=Endpoint("/orchestrator_/tasks/AppTasks/CreateAppTask"), - json=json_payload, - headers=header_folder(app_folder_key, app_folder_path), - ) + "FolderKey": UiPathConfig.folder_key, + "JobKey": UiPathConfig.job_key, + "ProcessKey": UiPathConfig.process_uuid, + }, + } def _normalize_priority(priority: str | None) -> str | None: @@ -196,6 +213,62 @@ def _normalize_priority(priority: str | None) -> str | None: return normalized +_TASK_TYPE_QUICKFORM = 6 + + +def _create_quickform_spec( + data: Optional[Dict[str, Any]], + title: str, + task_schema_key: str, + schema: Dict[str, Any], + creator_job_key: Optional[str] = None, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + priority: Optional[str] = None, + labels: Optional[List[str]] = None, + is_actionable_message_enabled: Optional[bool] = None, + actionable_message_metadata: Optional[Dict[str, Any]] = None, + source_name: str = "Agent", +) -> RequestSpec: + """Build the RequestSpec for Orchestrator's GenericTasks/CreateTask endpoint. + + Sets TaskType=QuickFormTask. Mirrors _create_spec but skips the AppTask-specific + shape (no appId, no action-schema-derived fieldSet/actionSet) and instead sends + taskSchemaKey + inline schema together. + + Both taskSchemaKey AND schema are sent on every call: the Agents runtime has no + Action Center package.uploaded subscriber populating the TaskSchemas table, so + Orchestrator upserts the schema (keyed by taskSchemaKey) and then creates the task + in the same call. + + Wire contract: UiPath/Orchestrator/src/Core/Application/Dto/Tasks/TaskCreateRequest.cs. + """ + json_payload: Dict[str, Any] = { + "type": _TASK_TYPE_QUICKFORM, + "taskSchemaKey": task_schema_key, + "schema": schema, + "title": title, + "data": data if data is not None else {}, + } + + if creator_job_key is not None: + json_payload["creatorJobKey"] = creator_job_key + + _apply_priority_labels_and_actionable_toggle( + json_payload, priority, labels, is_actionable_message_enabled + ) + if actionable_message_metadata is not None: + json_payload["actionableMessageMetaData"] = actionable_message_metadata + _apply_task_source(json_payload, source_name) + + return RequestSpec( + method="POST", + endpoint=Endpoint("/orchestrator_/tasks/GenericTasks/CreateTask"), + json=json_payload, + headers=header_folder(folder_key, folder_path), + ) + + def _retrieve_action_spec( action_key: str, app_folder_key: Optional[str], @@ -233,6 +306,34 @@ async def _assign_task_spec( } ] } + elif task_recipient.type == TaskRecipientType.WORKLOAD: + # This branch covers BOTH agent-side Workload criteria (single + # group, distributed by workload) AND agent-side CustomAssignees + # criteria (explicit email list — already resolved into + # `task_recipient.values` upstream). Both submit to the Action + # Center API as a "Workload" assignment; the difference is whether + # `values` carries one group or N emails. + request_spec.json = { + "taskAssignments": [ + { + "taskId": task_key, + "assignmentCriteria": "Workload", + "assigneeNamesOrEmails": task_recipient.values + or [recipient_value], + } + ] + } + elif task_recipient.type == TaskRecipientType.ROUND_ROBIN: + request_spec.json = { + "taskAssignments": [ + { + "taskId": task_key, + "assignmentCriteria": "RoundRobin", + "assigneeNamesOrEmails": task_recipient.values + or [recipient_value], + } + ] + } else: request_spec.json = { "taskAssignments": [ @@ -506,6 +607,154 @@ def create( ) return Task.model_validate(json_response) + @traced(name="tasks_create_quickform", run_type="uipath") + async def create_quickform_async( + self, + title: str, + task_schema_key: str, + schema: Dict[str, Any], + data: Optional[Dict[str, Any]] = None, + *, + folder_path: Optional[str] = None, + folder_key: Optional[str] = None, + assignee: Optional[str] = None, + recipient: Optional[TaskRecipient] = None, + priority: Optional[str] = None, + labels: Optional[List[str]] = None, + is_actionable_message_enabled: Optional[bool] = None, + actionable_message_metadata: Optional[Dict[str, Any]] = None, + creator_job_key: Optional[str] = None, + source_name: str = "Agent", + ) -> Task: + """Creates a new QuickForm task asynchronously. + + QuickForm tasks are schema-first HITL tasks rendered by FormLib in Action + Center. Both task_schema_key AND schema are required: the Agents runtime + does not pre-populate TaskSchemas via a package.uploaded subscriber, so + Orchestrator upserts the schema (keyed by task_schema_key) and creates + the task in the same call. + + Args: + title: The title of the task. + task_schema_key: UUID key of the schema. Used as the key under which + Orchestrator stores/looks up the schema in TaskSchemas. + schema: The HITL schema body to register/upsert. Sent inline on every + call. + data: Optional dictionary containing input data for the task. + folder_path: Optional folder path for the task. Required by the + Orchestrator controller (RequireOrganizationUnit) unless + folder_key is provided. + folder_key: Optional folder key, alternative to folder_path. + assignee: Optional username or email to assign the task to. + recipient: Optional structured recipient (user id / group id / + email). Resolved via identity service before assignment. + priority: Optional priority. Low / Medium / High / Critical. + labels: Optional list of labels for the task. + is_actionable_message_enabled: Whether actionable notifications are + enabled for this task. + actionable_message_metadata: Optional metadata override. For + QuickForm, when null, Orchestrator derives it from the + referenced TaskSchema. + creator_job_key: Optional. Identifies the job that triggered the + inline schema creation/upsert. + source_name: Source name on TaskSource. Defaults to 'Agent'. + + Returns: + Task: The created task object. + """ + spec = _create_quickform_spec( + title=title, + data=data, + task_schema_key=task_schema_key, + schema=schema, + creator_job_key=creator_job_key, + folder_key=folder_key, + folder_path=folder_path, + priority=priority, + labels=labels, + is_actionable_message_enabled=is_actionable_message_enabled, + actionable_message_metadata=actionable_message_metadata, + source_name=source_name, + ) + + response = await self.request_async( + spec.method, + spec.endpoint, + json=spec.json, + content=spec.content, + headers=spec.headers, + ) + json_response = response.json() + if assignee or recipient: + assign_spec = await _assign_task_spec( + self, json_response["id"], assignee, recipient + ) + await self.request_async( + assign_spec.method, + assign_spec.endpoint, + json=assign_spec.json, + content=assign_spec.content, + ) + return Task.model_validate(json_response) + + @traced(name="tasks_create_quickform", run_type="uipath") + def create_quickform( + self, + title: str, + task_schema_key: str, + schema: Dict[str, Any], + data: Optional[Dict[str, Any]] = None, + *, + folder_path: Optional[str] = None, + folder_key: Optional[str] = None, + assignee: Optional[str] = None, + recipient: Optional[TaskRecipient] = None, + priority: Optional[str] = None, + labels: Optional[List[str]] = None, + is_actionable_message_enabled: Optional[bool] = None, + actionable_message_metadata: Optional[Dict[str, Any]] = None, + creator_job_key: Optional[str] = None, + source_name: str = "Agent", + ) -> Task: + """Create a new QuickForm task synchronously. + + See :meth:`create_quickform_async` for parameter docs. + """ + spec = _create_quickform_spec( + title=title, + data=data, + task_schema_key=task_schema_key, + schema=schema, + creator_job_key=creator_job_key, + folder_key=folder_key, + folder_path=folder_path, + priority=priority, + labels=labels, + is_actionable_message_enabled=is_actionable_message_enabled, + actionable_message_metadata=actionable_message_metadata, + source_name=source_name, + ) + + response = self.request( + spec.method, + spec.endpoint, + json=spec.json, + content=spec.content, + headers=spec.headers, + ) + json_response = response.json() + if assignee or recipient: + assign_spec = asyncio.run( + _assign_task_spec(self, json_response["id"], assignee, recipient) + ) + self.request( + assign_spec.method, + assign_spec.endpoint, + json=assign_spec.json, + content=assign_spec.content, + ) + return Task.model_validate(json_response) + @resource_override( resource_type="app", resource_identifier="app_name", diff --git a/packages/uipath-platform/src/uipath/platform/action_center/tasks.py b/packages/uipath-platform/src/uipath/platform/action_center/tasks.py index f882cf40f..f1a932cb8 100644 --- a/packages/uipath-platform/src/uipath/platform/action_center/tasks.py +++ b/packages/uipath-platform/src/uipath/platform/action_center/tasks.py @@ -22,18 +22,35 @@ class TaskRecipientType(str, enum.Enum): GROUP_ID = "GroupId" EMAIL = "UserEmail" GROUP_NAME = "GroupName" + WORKLOAD = "Workload" + ROUND_ROBIN = "RoundRobin" class TaskRecipient(BaseModel): - """Model representing a task recipient.""" + """Model representing a task recipient. + + `value` is the single identifier (group name, group id, user id, email, …). + `values` is the multi-assignee form used by Workload-with-custom-emails + assignments; when set it takes precedence over `value` for the + `assigneeNamesOrEmails` payload. + + Note: there is no CustomAssignees member here on purpose. The agent-side + CustomAssignees criteria (AgentEscalationRecipientType.CUSTOM_ASSIGNEES, + type 11) is resolved to a Workload assignment with the explicit email list + in `values` before reaching this layer, so the Action Center + AssignTasks API only ever sees the existing literal types. + """ type: Literal[ TaskRecipientType.USER_ID, TaskRecipientType.GROUP_ID, TaskRecipientType.EMAIL, TaskRecipientType.GROUP_NAME, + TaskRecipientType.WORKLOAD, + TaskRecipientType.ROUND_ROBIN, ] = Field(..., alias="type") value: str = Field(..., alias="value") + values: Optional[List[str]] = Field(default=None, alias="values") display_name: Optional[str] = Field(default=None, alias="displayName") diff --git a/packages/uipath-platform/src/uipath/platform/agenthub/_remote_a2a_service.py b/packages/uipath-platform/src/uipath/platform/agenthub/_remote_a2a_service.py index c9475976c..037f4ff7d 100644 --- a/packages/uipath-platform/src/uipath/platform/agenthub/_remote_a2a_service.py +++ b/packages/uipath-platform/src/uipath/platform/agenthub/_remote_a2a_service.py @@ -7,12 +7,15 @@ import warnings from typing import Any, List +from urllib.parse import quote from ..common._base_service import BaseService +from ..common._bindings import resource_override from ..common._config import UiPathApiConfig from ..common._execution_context import UiPathExecutionContext from ..common._folder_context import FolderContext, header_folder from ..common._models import Endpoint, RequestSpec +from ..common._resource_identifier import resolve_retrieve_identifier from ..orchestrator import FolderService from .remote_a2a import RemoteA2aAgent @@ -149,19 +152,23 @@ async def main(): data = response.json() return [RemoteA2aAgent.model_validate(agent) for agent in data.get("value", [])] + @resource_override(resource_type="remoteA2aAgent", resource_identifier="name") + @resource_override(resource_type="remoteA2aAgent", resource_identifier="slug") def retrieve( self, - slug: str, + slug: str | None = None, *, + name: str | None = None, folder_path: str | None = None, ) -> RemoteA2aAgent: - """Retrieve a specific Remote A2A agent by slug. + """Retrieve a Remote A2A agent by its display name or legacy slug. .. warning:: This method is experimental and subject to change. Args: - slug: The unique slug identifier for the agent. + slug: The legacy slug identifier of the agent. + name: The display name of the agent. folder_path: The folder path where the agent is located. Returns: @@ -181,7 +188,8 @@ def retrieve( "remote_a2a.retrieve is experimental and subject to change.", stacklevel=2, ) - spec = self._retrieve_spec(slug=slug, folder_path=folder_path) + identifier = resolve_retrieve_identifier(name=name, slug=slug) + spec = self._retrieve_spec(name=identifier, folder_path=folder_path) response = self.request( spec.method, url=spec.endpoint, @@ -190,19 +198,23 @@ def retrieve( ) return RemoteA2aAgent.model_validate(response.json()) + @resource_override(resource_type="remoteA2aAgent", resource_identifier="name") + @resource_override(resource_type="remoteA2aAgent", resource_identifier="slug") async def retrieve_async( self, - slug: str, + slug: str | None = None, *, + name: str | None = None, folder_path: str | None = None, ) -> RemoteA2aAgent: - """Asynchronously retrieve a specific Remote A2A agent by slug. + """Asynchronously retrieve a Remote A2A agent by display name or legacy slug. .. warning:: This method is experimental and subject to change. Args: - slug: The unique slug identifier for the agent. + slug: The legacy slug identifier of the agent. + name: The display name of the agent. folder_path: The folder path where the agent is located. Returns: @@ -226,7 +238,8 @@ async def main(): "remote_a2a.retrieve_async is experimental and subject to change.", stacklevel=2, ) - spec = self._retrieve_spec(slug=slug, folder_path=folder_path) + identifier = resolve_retrieve_identifier(name=name, slug=slug) + spec = self._retrieve_spec(name=identifier, folder_path=folder_path) response = await self.request_async( spec.method, url=spec.endpoint, @@ -239,6 +252,13 @@ async def main(): def custom_headers(self) -> dict[str, str]: return self.folder_headers + def _resolve_folder_key(self, folder_path: str | None) -> str | None: + """Resolve folder key from folder_path, falling back to FolderContext.""" + if folder_path is not None: + return self._folders_service.retrieve_folder_key(folder_path) + + return self._folder_key + def _list_spec( self, *, @@ -269,14 +289,16 @@ def _list_spec( def _retrieve_spec( self, - slug: str, + name: str, *, folder_path: str | None, ) -> RequestSpec: - folder_key = self._folders_service.retrieve_folder_key(folder_path) + folder_key = self._resolve_folder_key(folder_path) return RequestSpec( method="GET", - endpoint=Endpoint(f"/agenthub_/api/remote-a2a-agents/{slug}"), + endpoint=Endpoint( + f"/agenthub_/api/remote-a2a-agents/{quote(name, safe='')}" + ), headers={ **header_folder(folder_key, None), }, diff --git a/packages/uipath-platform/src/uipath/platform/automation_ops/__init__.py b/packages/uipath-platform/src/uipath/platform/automation_ops/__init__.py new file mode 100644 index 000000000..87ce420ec --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/automation_ops/__init__.py @@ -0,0 +1,9 @@ +"""AutomationOps service package. + +Provides the ``AutomationOpsService`` client for retrieving deployed AI Trust +Layer policies from AgentHub. +""" + +from ._automation_ops_service import AutomationOpsService + +__all__ = ["AutomationOpsService"] diff --git a/packages/uipath-platform/src/uipath/platform/automation_ops/_automation_ops_service.py b/packages/uipath-platform/src/uipath/platform/automation_ops/_automation_ops_service.py new file mode 100644 index 000000000..b5eac8cdd --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/automation_ops/_automation_ops_service.py @@ -0,0 +1,70 @@ +"""AutomationOps service for UiPath Platform. + +Provides methods for retrieving deployed policies from the AgentHub service. +""" + +from typing import Any + +from uipath.core.tracing import traced + +from ..common._base_service import BaseService +from ..common._config import UiPathApiConfig +from ..common._execution_context import UiPathExecutionContext +from ..common._models import Endpoint, RequestSpec + +_DEPLOYED_POLICY_ENDPOINT = Endpoint("agenthub_/api/policies/deployed-policy") + + +class AutomationOpsService(BaseService): + """Service for interacting with UiPath AutomationOps policies via AgentHub.""" + + def __init__( + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + ) -> None: + super().__init__(config=config, execution_context=execution_context) + + @traced(name="automation_ops_get_deployed_policy", run_type="uipath") + def get_deployed_policy(self) -> dict[str, Any]: + """Retrieve the deployed policy. + + Returns: + The deployed policy response as a dictionary. Returns an empty + dict when no policy is deployed (empty 200 response body). + """ + spec = self._deployed_policy_spec() + response = self.request( + spec.method, + url=spec.endpoint, + headers=spec.headers, + scoped="tenant", + ) + if not response.content: + return {} + return response.json() + + @traced(name="automation_ops_get_deployed_policy", run_type="uipath") + async def get_deployed_policy_async(self) -> dict[str, Any]: + """Retrieve the deployed policy (async). + + Returns: + The deployed policy response as a dictionary. Returns an empty + dict when no policy is deployed (empty 200 response body). + """ + spec = self._deployed_policy_spec() + response = await self.request_async( + spec.method, + url=spec.endpoint, + headers=spec.headers, + scoped="tenant", + ) + if not response.content: + return {} + return response.json() + + def _deployed_policy_spec(self) -> RequestSpec: + return RequestSpec( + method="POST", + endpoint=_DEPLOYED_POLICY_ENDPOINT, + ) diff --git a/packages/uipath-platform/src/uipath/platform/chat/_llm_gateway_service.py b/packages/uipath-platform/src/uipath/platform/chat/_llm_gateway_service.py index d7c093d0d..65ec6223a 100644 --- a/packages/uipath-platform/src/uipath/platform/chat/_llm_gateway_service.py +++ b/packages/uipath-platform/src/uipath/platform/chat/_llm_gateway_service.py @@ -22,11 +22,14 @@ from pydantic import BaseModel from uipath.core.tracing import traced +from uipath.platform.constants import HEADER_AGENTHUB_CONFIG + from ..common._base_service import BaseService from ..common._config import UiPathApiConfig from ..common._endpoints_manager import EndpointManager from ..common._execution_context import UiPathExecutionContext from ..common._models import Endpoint +from ._model_capabilities import should_skip_temperature from .llm_gateway import ( ChatCompletion, SpecificToolChoice, @@ -35,6 +38,7 @@ ToolDefinition, ) from .llm_throttle import get_llm_semaphore +from .llm_trace_context import build_trace_context_headers # Common constants API_VERSION = "2024-10-21" # Standard API version for OpenAI-compatible endpoints @@ -58,7 +62,7 @@ def _build_llm_headers( "X-UiPath-LlmGateway-RequestingFeature": requesting_feature, } if agenthub_config: - headers["X-UiPath-AgentHub-Config"] = agenthub_config + headers[HEADER_AGENTHUB_CONFIG] = agenthub_config if action_id: headers["X-UiPath-LlmGateway-ActionId"] = action_id return headers @@ -172,6 +176,7 @@ def __init__( action_id: Optional[str] = None, ) -> None: super().__init__(config=config, execution_context=execution_context) + self._agenthub_config = agenthub_config self._llm_headers = _build_llm_headers( requesting_product, requesting_feature, agenthub_config, action_id ) @@ -224,7 +229,7 @@ async def embeddings( endpoint, json={"input": input}, params={"api-version": API_VERSION}, - headers=self._llm_headers, + headers={**self._llm_headers, **build_trace_context_headers()}, ) return TextEmbedding.model_validate(response.json()) @@ -324,12 +329,22 @@ class Country(BaseModel): ) endpoint = Endpoint("/" + endpoint) - request_body = { + is_reasoning_model = model.lower().startswith(("o1", "o3", "o4")) + + # Reasoning models (o1, o3, o4) don't support temperature; newer models + # reject it too, which only discovery knows about. + skip_temperature = is_reasoning_model or await should_skip_temperature( + self, model, self._agenthub_config + ) + + request_body: dict[str, Any] = { "messages": messages, "max_tokens": max_tokens, - "temperature": temperature, } + if not skip_temperature: + request_body["temperature"] = temperature + # Handle response_format - convert BaseModel to schema if needed if response_format: if isinstance(response_format, type) and issubclass( @@ -355,7 +370,7 @@ class Country(BaseModel): endpoint, json=request_body, params={"api-version": API_VERSION}, - headers=self._llm_headers, + headers={**self._llm_headers, **build_trace_context_headers()}, ) return ChatCompletion.model_validate(response.json()) @@ -384,6 +399,7 @@ def __init__( action_id: Optional[str] = None, ) -> None: super().__init__(config=config, execution_context=execution_context) + self._agenthub_config = agenthub_config self._llm_headers = _build_llm_headers( requesting_product, requesting_feature, agenthub_config, action_id ) @@ -400,7 +416,7 @@ async def chat_completions( presence_penalty: float = 0, top_p: float | None = 1, top_k: int | None = None, - tools: list[ToolDefinition] | None = None, + tools: list[ToolDefinition | dict[str, Any]] | None = None, tool_choice: ToolChoice | None = None, response_format: dict[str, Any] | type[BaseModel] | None = None, api_version: str = NORMALIZED_API_VERSION, @@ -435,9 +451,11 @@ async def chat_completions( Controls diversity by considering only the top p probability mass. Defaults to 1. top_k (int, optional): Nucleus sampling parameter. Controls diversity by considering only the top k most probable tokens. Defaults to None. - tools (Optional[List[ToolDefinition]], optional): List of tool definitions that the - model can call. Tools enable the model to perform actions or retrieve information - beyond text generation. Defaults to None. + tools (Optional[List[ToolDefinition | dict]], optional): List of tool definitions + that the model can call. Tools enable the model to perform actions or retrieve + information beyond text generation. A tool given as a dict must already be in + UiPath wire format and is forwarded unchanged, which allows arbitrary nested + JSON schemas in its parameters. Defaults to None. tool_choice (Optional[ToolChoice], optional): Controls which tools the model can call. Can be "auto" (model decides), "none" (no tools), or a specific tool choice. Defaults to None. @@ -543,17 +561,27 @@ class Country(BaseModel): ) endpoint = Endpoint("/" + endpoint) - # Build request body - Claude models don't support some OpenAI-specific parameters - is_claude_model = "claude" in model.lower() + # Build request body - some models don't support certain parameters + model_lower = model.lower() + is_claude_model = "claude" in model_lower + is_reasoning_model = model_lower.startswith(("o1", "o3", "o4")) + + # Reasoning models (o1, o3, o4) don't support temperature; newer models + # reject it too, which only discovery knows about. + skip_temperature = is_reasoning_model or await should_skip_temperature( + self, model, self._agenthub_config + ) - request_body = { + request_body: dict[str, Any] = { "messages": converted_messages, "max_tokens": max_tokens, - "temperature": temperature, } - # Only add OpenAI-specific parameters for non-Claude models - if not is_claude_model: + if not skip_temperature: + request_body["temperature"] = temperature + + # Only add OpenAI-specific parameters for non-Claude and non-reasoning models + if not is_claude_model and not is_reasoning_model: request_body["n"] = n request_body["frequency_penalty"] = frequency_penalty request_body["presence_penalty"] = presence_penalty @@ -582,10 +610,15 @@ class Country(BaseModel): # Use provided dictionary format directly request_body["response_format"] = response_format - # Add tools if provided - convert to UiPath format + # Add tools if provided. A tool already in UiPath wire format (a dict) is + # passed through unchanged so callers can supply an arbitrary JSON schema + # for the parameters; ToolDefinition objects are converted as before. if tools: request_body["tools"] = [ - self._convert_tool_to_uipath_format(tool) for tool in tools + tool + if isinstance(tool, dict) + else self._convert_tool_to_uipath_format(tool) + for tool in tools ] # Handle tool_choice @@ -599,6 +632,7 @@ class Country(BaseModel): headers = { **self._llm_headers, + **build_trace_context_headers(), "X-UiPath-LlmGateway-NormalizedApi-ModelName": model, "X-UiPath-LLMGateway-AllowFull4xxResponse": "true", } diff --git a/packages/uipath-platform/src/uipath/platform/chat/_model_capabilities.py b/packages/uipath-platform/src/uipath/platform/chat/_model_capabilities.py new file mode 100644 index 000000000..4a435d117 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/chat/_model_capabilities.py @@ -0,0 +1,98 @@ +"""Per-model parameter support, resolved from the LLM Gateway discovery API. + +Newer models reject parameters that older ones accept: passing `temperature` to +a model that dropped it fails the whole request with HTTP 400. The gateway +publishes these constraints per model under `modelDetails`, so callers can strip +the parameter instead of guessing from the model name. +""" + +import logging +from typing import Any + +from ..common._base_service import BaseService +from ..common._endpoints_manager import EndpointManager +from ..common._models import Endpoint +from ..constants import HEADER_AGENTHUB_CONFIG + +logger = logging.getLogger(__name__) + +CacheKey = tuple[str, str | None] + +# One fetch per (base_url, agenthub_config); an empty mapping caches "couldn't read +# discovery". Deliberately unlocked: a racing duplicate GET is cheaper than an +# asyncio.Lock pinned to whichever event loop touched it first. +_model_details: dict[CacheKey, dict[str, dict[str, Any]]] = {} + + +async def should_skip_temperature( + service: BaseService, model: str, agenthub_config: str | None +) -> bool: + """Whether `model` rejects the `temperature` parameter. + + Args: + service: Service used to call discovery; supplies base URL and auth. + model: Model name as sent to the gateway. + agenthub_config: AgentHub config scoping which models are visible. + + Returns: + True only when discovery explicitly reports the model skips temperature. + Unknown models, unreachable discovery, and any unexpected failure return + False, leaving the caller's request untouched. + """ + try: + details = await _details_for(service, model, agenthub_config) + except Exception as e: + logger.warning( + "Could not resolve parameter support for model %s (%s); " + "sending model parameters unchanged", + model, + e, + ) + return False + + return bool(details.get("shouldSkipTemperature", False)) + + +async def _details_for( + service: BaseService, model: str, agenthub_config: str | None +) -> dict[str, Any]: + key = (service._config.base_url, agenthub_config) + + if key not in _model_details: + _model_details[key] = await _fetch_model_details(service, agenthub_config) + + return _model_details[key].get(model, {}) + + +async def _fetch_model_details( + service: BaseService, agenthub_config: str | None +) -> dict[str, dict[str, Any]]: + headers = {HEADER_AGENTHUB_CONFIG: agenthub_config} if agenthub_config else {} + endpoint = Endpoint("/" + EndpointManager.get_discovery_endpoint()) + + try: + response = await service.request_async("GET", endpoint, headers=headers) + models = response.json() + except Exception as e: + # Falling back to the caller's parameters is the pre-existing behaviour; + # failing the LLM call because discovery is down would be worse. + logger.warning( + "LLM Gateway discovery unavailable (%s); sending model parameters unchanged", + e, + ) + return {} + + if not isinstance(models, list): + logger.warning("Unexpected LLM Gateway discovery payload; expected a list") + return {} + + return { + model["modelName"]: model.get("modelDetails") or {} + for model in models + if isinstance(model, dict) and model.get("modelName") + } + + +def _reset_cache() -> None: + """Clear the discovery cache. For tests.""" + _model_details.clear() diff --git a/packages/uipath-platform/src/uipath/platform/chat/llm_trace_context.py b/packages/uipath-platform/src/uipath/platform/chat/llm_trace_context.py new file mode 100644 index 000000000..c10047669 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/chat/llm_trace_context.py @@ -0,0 +1,58 @@ +"""W3C-style trace context headers for LLM Gateway requests.""" + +from opentelemetry import trace +from uipath.core.feature_flags import FeatureFlags +from uipath.core.tracing.span_utils import UiPathSpanUtils + +from ..common._config import UiPathConfig +from ..common._span_utils import _SpanUtils, resolve_project_id + + +def build_trace_context_headers( + extra_baggage: list[str] | None = None, +) -> dict[str, str]: + """Build W3C-style trace context headers for LLM Gateway requests. + + Resolves the current span via ``UiPathSpanUtils.get_external_current_span()`` + (which returns the deepest active span from the LLMOps hierarchy) with a + fallback to ``trace.get_current_span()``. + + Args: + extra_baggage: Additional baggage entries (e.g. ``["source=agents"]``) + that callers can inject alongside the platform-level entries. + + Returns an empty dict when the ``EnableTraceContextHeaders`` feature flag + is not enabled, or when no active span is present. + """ + if not FeatureFlags.is_flag_enabled("EnableTraceContextHeaders"): + return {} + + headers: dict[str, str] = {} + llmops_span = UiPathSpanUtils.get_external_current_span() + span = llmops_span or trace.get_current_span() + config_trace_id = UiPathConfig.trace_id + ctx = span.get_span_context() + if config_trace_id and ctx and ctx.span_id: + trace_id = _SpanUtils.normalize_trace_id(config_trace_id) + # An OTEL span id is 64-bit => 16 lowercase hex chars, and the W3C traceparent + # requires the trailing trace-flags segment. The LLM Gateway's strict parser + # (UiPath.Tracing.TraceParent.TryParse) rejects the header unless it is exactly + # {version}-{32-hex trace}-{16-hex span}-{2-hex flags}; on rejection the gateway + # synthesizes a fresh root trace and drops inbound baggage, orphaning the audit + # span from the caller's trace. Emitting 32-hex span / no flags was the bug. + span_id = format(ctx.span_id, "016x") + headers["x-uipath-traceparent-id"] = f"00-{trace_id}-{span_id}-01" + + baggage_parts: list[str] = list(extra_baggage) if extra_baggage else [] + if folder_key := UiPathConfig.folder_key: + baggage_parts.append(f"folderKey={folder_key}") + if agent_id := resolve_project_id(): + baggage_parts.append(f"agentId={agent_id}") + if process_uuid := UiPathConfig.process_uuid: + baggage_parts.append(f"processKey={process_uuid}") + if job_key := UiPathConfig.job_key: + baggage_parts.append(f"jobKey={job_key}") + if baggage_parts: + headers["x-uipath-tracebaggage"] = ",".join(baggage_parts) + + return headers diff --git a/packages/uipath-platform/src/uipath/platform/common/__init__.py b/packages/uipath-platform/src/uipath/platform/common/__init__.py index 40fc1ac34..802ec67bc 100644 --- a/packages/uipath-platform/src/uipath/platform/common/__init__.py +++ b/packages/uipath-platform/src/uipath/platform/common/__init__.py @@ -3,10 +3,13 @@ This module contains common models used across multiple services. """ +from uipath.core.triggers import UiPathResumeMetadata + from ._api_client import ApiClient -from ._base_service import BaseService +from ._base_service import BaseService, resolve_trace_id from ._bindings import ( ConnectionResourceOverwrite, + EntityResourceOverwrite, GenericResourceOverwrite, ResourceOverwrite, ResourceOverwriteParser, @@ -15,13 +18,25 @@ ) from ._config import UiPathApiConfig, UiPathConfig from ._endpoints_manager import EndpointManager -from ._execution_context import UiPathExecutionContext -from ._external_application_service import ExternalApplicationService +from ._execution_context import ExecutionSourceContext, UiPathExecutionContext from ._folder_context import FolderContext, header_folder from ._http_config import get_ca_bundle_path, get_httpx_client_kwargs from ._models import Endpoint, RequestSpec +from ._reference_context import ( + ReferenceContext, + ReferenceContextAccessor, + ReferenceEntry, +) from ._service_url_overrides import inject_routing_headers, resolve_service_url -from ._span_utils import UiPathSpan, _SpanUtils +from ._span_utils import ( + ExecutionType, + ReferenceHierarchySpanProcessor, + SpanSource, + SpanStatus, + UiPathSpan, + VerbosityLevel, + _SpanUtils, +) from ._url import UiPathUrl from ._user_agent import user_agent_value from .auth import TokenData @@ -47,19 +62,27 @@ WaitEphemeralIndex, WaitEphemeralIndexRaw, WaitEscalation, + WaitIntegrationEvent, WaitJob, WaitJobRaw, WaitSystemAgent, WaitTask, + WaitUntil, ) from .paging import PagedResult +from .timeout import ( + UiPathTimeoutError, + assert_no_timeout, + get_resume_metadata, + is_timeout, +) __all__ = [ "ApiClient", "BaseService", "UiPathApiConfig", "UiPathExecutionContext", - "ExternalApplicationService", + "ExecutionSourceContext", "FolderContext", "TokenData", "UiPathConfig", @@ -88,6 +111,8 @@ "WaitEphemeralIndexRaw", "DocumentExtractionValidation", "WaitDocumentExtractionValidation", + "WaitIntegrationEvent", + "WaitUntil", "RequestSpec", "Endpoint", "UiPathUrl", @@ -100,14 +125,29 @@ "EndpointManager", "jsonschema_to_pydantic", "ConnectionResourceOverwrite", + "EntityResourceOverwrite", + "ExecutionType", "GenericResourceOverwrite", "ResourceOverwrite", "ResourceOverwriteParser", "ResourceOverwritesContext", + "ReferenceEntry", + "ReferenceContext", + "ReferenceContextAccessor", + "ReferenceHierarchySpanProcessor", + "SpanSource", + "SpanStatus", "UiPathSpan", + "VerbosityLevel", "_SpanUtils", "resolve_service_url", "inject_routing_headers", + "resolve_trace_id", + "UiPathTimeoutError", + "UiPathResumeMetadata", + "assert_no_timeout", + "get_resume_metadata", + "is_timeout", ] from .validation import validate_pagination_params diff --git a/packages/uipath-platform/src/uipath/platform/common/_base_service.py b/packages/uipath-platform/src/uipath/platform/common/_base_service.py index d236e4ef5..8db2a51d1 100644 --- a/packages/uipath-platform/src/uipath/platform/common/_base_service.py +++ b/packages/uipath-platform/src/uipath/platform/common/_base_service.py @@ -3,6 +3,7 @@ from logging import getLogger from typing import Any, Literal, Union +from anyio import to_thread from httpx import ( URL, AsyncClient, @@ -20,6 +21,8 @@ stop_after_attempt, ) +from uipath.platform.constants import HEADER_USER_AGENT + from ..errors import EnrichedException from ._config import UiPathApiConfig from ._execution_context import UiPathExecutionContext @@ -27,7 +30,6 @@ from ._service_url_overrides import inject_routing_headers, resolve_service_url from ._url import UiPathUrl from ._user_agent import user_agent_value -from .constants import HEADER_USER_AGENT from .retry import ( MAX_RETRY_ATTEMPTS, is_retryable_platform_exception, @@ -66,14 +68,87 @@ def _get_caller_component() -> str: _TRACE_PARENT_HEADER = "x-uipath-traceparent-id" +def resolve_trace_id(fallback: str | None = None) -> str | None: + """Resolve the current UiPath trace id as a 32-char hex string. + + Same lookup chain :func:`_inject_trace_context` uses to compose the + ``x-uipath-traceparent-id`` header, exposed as a public helper so + callers can capture the value when they need it in a request body + (e.g. governance compensation) or before hopping to a background + thread that won't inherit the OpenTelemetry context. + + Resolution order (first hit wins): + + 1. :attr:`UiPathConfig.trace_id` (``UIPATH_TRACE_ID`` env var), + normalized via :meth:`_SpanUtils.normalize_trace_id`. This is the + canonical agent trace id the LLMOps exporter binds spans to. + 2. The LLMOps external span trace id, when a provider is registered + via :meth:`UiPathSpanUtils.register_current_span_provider`. + 3. The current OpenTelemetry span trace id. + 4. The caller-supplied ``fallback``. + + Args: + fallback: Returned when nothing above resolves. + + Returns: + Lower-case 32-char hex trace id, or ``fallback`` (which may be + ``None``) when no source yields a usable value. + + Thread Safety: + Steps 2 and 3 read OpenTelemetry's thread-local context. Call this + on the thread that owns the live span (e.g. the agent's hook + thread) and capture the result before submitting work to a + background pool — worker threads do not inherit the context. + """ + from uipath.core.tracing.span_utils import UiPathSpanUtils + + from ._config import UiPathConfig + from ._span_utils import _SpanUtils + + config_trace_id = UiPathConfig.trace_id + if config_trace_id: + try: + return _SpanUtils.normalize_trace_id(config_trace_id) + except ValueError: + # Malformed UIPATH_TRACE_ID — fall through to OTel context. + pass + + llmops_span = UiPathSpanUtils.get_external_current_span() + span = llmops_span or trace.get_current_span() + ctx = span.get_span_context() + if ctx.trace_id: + return format_trace_id(ctx.trace_id) + + return fallback + + def _inject_trace_context(headers: dict[str, str]) -> None: - """Inject UiPath trace context header from the active OTEL span.""" - span = trace.get_current_span() + """Inject UiPath trace context header. + + Trace ID: uses the agent trace ID from UIPATH_TRACE_ID env var (same + remapping the LLMOps exporter applies), falling back to the OTEL trace ID. + Span ID: uses the LLMOps tool span (via external span provider) so the + span ID matches what's visible in the LLMOps trace UI. + """ + from uipath.core.tracing.span_utils import UiPathSpanUtils + + from ._config import UiPathConfig + from ._span_utils import _SpanUtils + + llmops_span = UiPathSpanUtils.get_external_current_span() + span = llmops_span or trace.get_current_span() ctx = span.get_span_context() - if ctx.trace_id and ctx.span_id: - headers[_TRACE_PARENT_HEADER] = ( - f"00-{format_trace_id(ctx.trace_id)}-{format_span_id(ctx.span_id)}-01" - ) + if not (ctx.trace_id and ctx.span_id): + return + + config_trace_id = UiPathConfig.trace_id + trace_id = ( + _SpanUtils.normalize_trace_id(config_trace_id) + if config_trace_id + else format_trace_id(ctx.trace_id) + ) + span_id = format_span_id(ctx.span_id) + headers[_TRACE_PARENT_HEADER] = f"00-{trace_id}-{span_id}-01" class BaseService: @@ -97,6 +172,18 @@ def __init__( super().__init__() + async def aclose(self) -> None: + """Close the HTTP clients owned by this service. + + The asynchronous client is closed by the active async backend. Closing the + synchronous client can block while its transport is torn down, so that work + runs in a backend-neutral worker thread. + """ + try: + await self._client_async.aclose() + finally: + await to_thread.run_sync(self._client.close) + @retry( retry=( retry_if_exception(is_retryable_platform_exception) diff --git a/packages/uipath-platform/src/uipath/platform/common/_bindings.py b/packages/uipath-platform/src/uipath/platform/common/_bindings.py index 449d2a7ef..a93880896 100644 --- a/packages/uipath-platform/src/uipath/platform/common/_bindings.py +++ b/packages/uipath-platform/src/uipath/platform/common/_bindings.py @@ -14,7 +14,14 @@ Union, ) -from pydantic import AliasChoices, BaseModel, ConfigDict, Field, TypeAdapter +from pydantic import ( + AliasChoices, + BaseModel, + ConfigDict, + Field, + TypeAdapter, + model_validator, +) logger = logging.getLogger(__name__) @@ -45,7 +52,15 @@ def folder_identifier(self) -> str: class GenericResourceOverwrite(ResourceOverwrite): resource_type: Literal[ - "process", "index", "app", "asset", "bucket", "mcpServer", "queue", "entity" + "process", + "index", + "app", + "asset", + "bucket", + "mcpServer", + "queue", + "remoteA2aAgent", + "memorySpace", ] name: str = Field(alias="name") folder_path: str = Field(alias="folderPath") @@ -59,6 +74,29 @@ def folder_identifier(self) -> str: return self.folder_path +class EntityResourceOverwrite(ResourceOverwrite): + resource_type: Literal["entity"] + name: str = Field(alias="name") + folder_id: Optional[str] = Field(default=None, alias="folderId") + folder_path: Optional[str] = Field(default=None, alias="folderPath") + + @model_validator(mode="after") + def validate_folder_identifier(self) -> "EntityResourceOverwrite": + if self.folder_id and self.folder_path: + raise ValueError("Only one of folderId or folderPath may be provided.") + if not self.folder_id and not self.folder_path: + raise ValueError("Either folderId or folderPath must be provided.") + return self + + @property + def resource_identifier(self) -> str: + return self.name + + @property + def folder_identifier(self) -> str: + return self.folder_id or self.folder_path or "" + + class ConnectionResourceOverwrite(ResourceOverwrite): resource_type: Literal["connection"] # In eval context, studio web provides "ConnectionId". @@ -83,7 +121,9 @@ def folder_identifier(self) -> str: ResourceOverwriteUnion = Annotated[ - Union[GenericResourceOverwrite, ConnectionResourceOverwrite], + Union[ + GenericResourceOverwrite, EntityResourceOverwrite, ConnectionResourceOverwrite + ], Field(discriminator="resource_type"), ] @@ -112,9 +152,23 @@ def parse(cls, key: str, value: dict[str, Any]) -> ResourceOverwrite: The appropriate ResourceOverwrite subclass instance """ resource_type = key.split(".")[0] - value_with_type = {"resource_type": resource_type, **value} + normalized_value = cls._normalize_value(resource_type, value) + value_with_type = {"resource_type": resource_type, **normalized_value} return cls._adapter.validate_python(value_with_type) + @staticmethod + def _normalize_value(resource_type: str, value: dict[str, Any]) -> dict[str, Any]: + if resource_type != "entity": + return value + + normalized = dict(value) + if "folderId" in normalized: + normalized["folder_id"] = normalized.pop("folderId") + if "folderPath" in normalized: + normalized["folder_path"] = normalized.pop("folderPath") + + return normalized + _resource_overwrites: ContextVar[Optional[dict[str, ResourceOverwrite]]] = ContextVar( "resource_overwrites", default=None diff --git a/packages/uipath-platform/src/uipath/platform/common/_config.py b/packages/uipath-platform/src/uipath/platform/common/_config.py index b656830f6..549844db8 100644 --- a/packages/uipath-platform/src/uipath/platform/common/_config.py +++ b/packages/uipath-platform/src/uipath/platform/common/_config.py @@ -36,13 +36,13 @@ def __repr__(self) -> str: @property def bindings_file_path(self) -> Path: - from uipath.platform.common.constants import UIPATH_BINDINGS_FILE + from uipath.platform.constants import UIPATH_BINDINGS_FILE return Path(UIPATH_BINDINGS_FILE) @property def config_file_path(self) -> Path: - from uipath.platform.common.constants import ( + from uipath.platform.constants import ( ENV_UIPATH_CONFIG_PATH, UIPATH_CONFIG_FILE, ) @@ -51,73 +51,97 @@ def config_file_path(self) -> Path: @property def config_file_name(self) -> str: - from uipath.platform.common.constants import UIPATH_CONFIG_FILE + from uipath.platform.constants import UIPATH_CONFIG_FILE return UIPATH_CONFIG_FILE @property def project_id(self) -> str | None: - from uipath.platform.common.constants import ENV_UIPATH_PROJECT_ID + from uipath.platform.constants import ENV_UIPATH_PROJECT_ID return os.getenv(ENV_UIPATH_PROJECT_ID, None) + @property + def agent_id(self) -> str | None: + from uipath.platform.constants import ENV_UIPATH_AGENT_ID + + return os.getenv(ENV_UIPATH_AGENT_ID) or self.project_id + + @property + def cloud_user_id(self) -> str | None: + from uipath.platform.constants import ENV_UIPATH_CLOUD_USER_ID + + return os.getenv(ENV_UIPATH_CLOUD_USER_ID, None) + + @property + def project_files_source(self) -> str | None: + from uipath.platform.constants import ENV_UIPATH_PROJECT_FILES_SOURCE + + return os.getenv(ENV_UIPATH_PROJECT_FILES_SOURCE, None) + @property def project_key(self) -> str | None: - from uipath.platform.common.constants import ENV_PROJECT_KEY + from uipath.platform.constants import ENV_PROJECT_KEY return os.getenv(ENV_PROJECT_KEY, None) @property def tenant_name(self) -> str | None: - from uipath.platform.common.constants import ENV_TENANT_NAME + from uipath.platform.constants import ENV_TENANT_NAME return os.getenv(ENV_TENANT_NAME, None) @property def tenant_id(self) -> str | None: - from uipath.platform.common.constants import ENV_TENANT_ID + from uipath.platform.constants import ENV_TENANT_ID return os.getenv(ENV_TENANT_ID, None) @property def organization_id(self) -> str | None: - from uipath.platform.common.constants import ENV_ORGANIZATION_ID + from uipath.platform.constants import ENV_ORGANIZATION_ID return os.getenv(ENV_ORGANIZATION_ID, None) @property def base_url(self) -> str | None: - from uipath.platform.common.constants import ENV_BASE_URL + from uipath.platform.constants import ENV_BASE_URL return os.getenv(ENV_BASE_URL, None) @property def folder_key(self) -> str | None: - from uipath.platform.common.constants import ENV_FOLDER_KEY + from uipath.platform.constants import ENV_FOLDER_KEY return os.getenv(ENV_FOLDER_KEY, None) @property def folder_path(self) -> str | None: - from uipath.platform.common.constants import ENV_FOLDER_PATH + from uipath.platform.constants import ENV_FOLDER_PATH return os.getenv(ENV_FOLDER_PATH, None) + @property + def process_key(self) -> str | None: + from uipath.platform.constants import ENV_PROCESS_KEY + + return os.getenv(ENV_PROCESS_KEY, None) + @property def process_uuid(self) -> str | None: - from uipath.platform.common.constants import ENV_UIPATH_PROCESS_UUID + from uipath.platform.constants import ENV_UIPATH_PROCESS_UUID return os.getenv(ENV_UIPATH_PROCESS_UUID, None) @property def trace_id(self) -> str | None: - from uipath.platform.common.constants import ENV_UIPATH_TRACE_ID + from uipath.platform.constants import ENV_UIPATH_TRACE_ID return os.getenv(ENV_UIPATH_TRACE_ID, None) @property def process_version(self) -> str | None: - from uipath.platform.common.constants import ENV_UIPATH_PROCESS_VERSION + from uipath.platform.constants import ENV_UIPATH_PROCESS_VERSION return os.getenv(ENV_UIPATH_PROCESS_VERSION, None) @@ -127,39 +151,39 @@ def is_studio_project(self) -> bool: @property def job_key(self) -> str | None: - from uipath.platform.common.constants import ENV_JOB_KEY + from uipath.platform.constants import ENV_JOB_KEY return os.getenv(ENV_JOB_KEY, None) @property def has_legacy_eval_folder(self) -> bool: - from uipath.platform.common.constants import LEGACY_EVAL_FOLDER + from uipath.platform.constants import LEGACY_EVAL_FOLDER eval_path = Path(os.getcwd()) / LEGACY_EVAL_FOLDER return eval_path.exists() and eval_path.is_dir() @property def has_eval_folder(self) -> bool: - from uipath.platform.common.constants import EVALS_FOLDER + from uipath.platform.constants import EVALS_FOLDER coded_eval_path = Path(os.getcwd()) / EVALS_FOLDER return coded_eval_path.exists() and coded_eval_path.is_dir() @property def entry_points_file_path(self) -> Path: - from uipath.platform.common.constants import ENTRY_POINTS_FILE + from uipath.platform.constants import ENTRY_POINTS_FILE return Path(ENTRY_POINTS_FILE) @property def uiproj_file_path(self) -> Path: - from uipath.platform.common.constants import UIPROJ_FILE + from uipath.platform.constants import UIPROJ_FILE return Path(UIPROJ_FILE) @property def studio_metadata_file_path(self) -> Path: - from uipath.platform.common.constants import STUDIO_METADATA_FILE + from uipath.platform.constants import STUDIO_METADATA_FILE return Path(".uipath", STUDIO_METADATA_FILE) @@ -174,7 +198,7 @@ def is_rooted_to_debug_job(self) -> bool: @property def is_tracing_enabled(self) -> bool: - from uipath.platform.common.constants import ENV_TRACING_ENABLED + from uipath.platform.constants import ENV_TRACING_ENABLED return os.getenv(ENV_TRACING_ENABLED, "true").lower() == "true" diff --git a/packages/uipath-platform/src/uipath/platform/common/_endpoints_manager.py b/packages/uipath-platform/src/uipath/platform/common/_endpoints_manager.py index 044ab1393..8e3b436ac 100644 --- a/packages/uipath-platform/src/uipath/platform/common/_endpoints_manager.py +++ b/packages/uipath-platform/src/uipath/platform/common/_endpoints_manager.py @@ -21,6 +21,7 @@ class UiPathEndpoints(Enum): "agenthub_/llm/raw/vendor/{vendor}/model/{model}/completions" ) AH_CAPABILITIES_ENDPOINT = "agenthub_/llm/api/capabilities" + AH_DISCOVERY_ENDPOINT = "agenthub_/llm/api/discovery" OR_NORMALIZED_COMPLETION_ENDPOINT = "orchestrator_/llm/api/chat/completions" OR_PASSTHROUGH_COMPLETION_ENDPOINT = "orchestrator_/llm/openai/deployments/{model}/chat/completions?api-version={api_version}" @@ -29,6 +30,7 @@ class UiPathEndpoints(Enum): "orchestrator_/llm/raw/vendor/{vendor}/model/{model}/completions" ) OR_CAPABILITIES_ENDPOINT = "orchestrator_/llm/api/capabilities" + OR_DISCOVERY_ENDPOINT = "orchestrator_/llm/api/discovery" class EndpointManager: @@ -185,6 +187,14 @@ def get_normalized_endpoint(cls) -> str: UiPathEndpoints.OR_NORMALIZED_COMPLETION_ENDPOINT, ) + @classmethod + def get_discovery_endpoint(cls) -> str: + """Get the model discovery endpoint.""" + return cls._select_endpoint( + UiPathEndpoints.AH_DISCOVERY_ENDPOINT, + UiPathEndpoints.OR_DISCOVERY_ENDPOINT, + ) + @classmethod def get_embeddings_endpoint(cls) -> str: """Get the embeddings endpoint.""" diff --git a/packages/uipath-platform/src/uipath/platform/common/_execution_context.py b/packages/uipath-platform/src/uipath/platform/common/_execution_context.py index de54c0c99..4106afc1e 100644 --- a/packages/uipath-platform/src/uipath/platform/common/_execution_context.py +++ b/packages/uipath-platform/src/uipath/platform/common/_execution_context.py @@ -1,6 +1,33 @@ +from contextvars import ContextVar, Token from os import environ as env -from uipath.platform.common.constants import ENV_JOB_ID, ENV_JOB_KEY, ENV_ROBOT_KEY +from uipath.platform.constants import ENV_JOB_ID, ENV_JOB_KEY, ENV_ROBOT_KEY + +_execution_source: ContextVar[str | None] = ContextVar("execution_source", default=None) + + +class ExecutionSourceContext: + """Scope the execution source for the duration of a run. + + Carries the source (e.g. ``runtime``/``playground``/``eval``) via a context + variable and releases it on exit so it stays correctly scoped in concurrent + runs. The CLI enters this with ``UiPathRuntimeContext.execution_source`` so + platform clients can read it via + :attr:`UiPathExecutionContext.execution_source`. + """ + + def __init__(self, execution_source: str | None) -> None: + self._execution_source = execution_source + self._token: Token[str | None] | None = None + + def __enter__(self) -> "ExecutionSourceContext": + self._token = _execution_source.set(self._execution_source) + return self + + def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: + if self._token is not None: + _execution_source.reset(self._token) + self._token = None class UiPathExecutionContext: @@ -76,3 +103,13 @@ def robot_key(self) -> str | None: raise ValueError(f"Robot key is not set ({ENV_ROBOT_KEY})") return self._robot_key + + @property + def execution_source(self) -> str | None: + """Get the execution source for the current run. + + Identifies the run context (e.g. ``runtime``/``playground``/``eval``), + derived from the CLI command and carried via + :class:`ExecutionSourceContext`. Returns ``None`` when not set. + """ + return _execution_source.get() diff --git a/packages/uipath-platform/src/uipath/platform/common/_folder_context.py b/packages/uipath-platform/src/uipath/platform/common/_folder_context.py index 4d401aded..6adc9af77 100644 --- a/packages/uipath-platform/src/uipath/platform/common/_folder_context.py +++ b/packages/uipath-platform/src/uipath/platform/common/_folder_context.py @@ -2,7 +2,7 @@ from os import environ as env from typing import Any, Optional -from uipath.platform.common.constants import ( +from uipath.platform.constants import ( ENV_FOLDER_KEY, ENV_FOLDER_PATH, HEADER_FOLDER_KEY, diff --git a/packages/uipath-platform/src/uipath/platform/common/_http_config.py b/packages/uipath-platform/src/uipath/platform/common/_http_config.py index 191d768c4..a367db7a5 100644 --- a/packages/uipath-platform/src/uipath/platform/common/_http_config.py +++ b/packages/uipath-platform/src/uipath/platform/common/_http_config.py @@ -66,8 +66,9 @@ def get_httpx_client_kwargs( ca_bundle = get_ca_bundle_path() client_kwargs["verify"] = create_ssl_context(ca_bundle) if ca_bundle else False + from uipath.platform.constants import HEADER_LICENSING_CONTEXT + from ._config import UiPathConfig - from .constants import HEADER_LICENSING_CONTEXT merged_headers: Dict[str, str] = {} licensing_context = UiPathConfig.licensing_context diff --git a/packages/uipath-platform/src/uipath/platform/common/_job_context.py b/packages/uipath-platform/src/uipath/platform/common/_job_context.py new file mode 100644 index 000000000..b312adb73 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/common/_job_context.py @@ -0,0 +1,14 @@ +from uipath.platform.constants import HEADER_JOB_KEY + +from ._config import UiPathConfig + + +def header_job_key() -> dict[str, str]: + """Return the X-UiPath-JobKey header when the orchestrator job key is set. + + Returns an empty dict when ``UiPathConfig.job_key`` is unset or empty. + """ + job_key = UiPathConfig.job_key + if not job_key: + return {} + return {HEADER_JOB_KEY: job_key} diff --git a/packages/uipath-platform/src/uipath/platform/common/_reference_context.py b/packages/uipath-platform/src/uipath/platform/common/_reference_context.py new file mode 100644 index 000000000..648d25e69 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/common/_reference_context.py @@ -0,0 +1,264 @@ +"""Immutable reference-hierarchy context for span propagation. + +Follows the same design as service-common BaggageContext: +- Immutable, copy-on-write — each mutating call returns a NEW instance so + sibling spans cannot bleed context into each other. +- ContextVar-backed accessor — flows across await boundaries without + threading the value through every function signature. +- Wire format compatible with the ``ref.*`` keys in ``x-uipath-tracebaggage`` + so context parsed by service-common middleware is understood here and + vice-versa. +""" + +from __future__ import annotations + +import contextvars +import uuid +from dataclasses import dataclass +from typing import ClassVar, Dict, Iterator, List, Optional, Tuple + +__all__ = [ + "ReferenceEntry", + "ReferenceContext", + "ReferenceContextAccessor", + "BAGGAGE_HEADER_NAME", + "BAGGAGE_KEY_TYPE", + "BAGGAGE_KEY_ID", + "BAGGAGE_KEY_VERSION", +] + +BAGGAGE_HEADER_NAME = "x-uipath-tracebaggage" + +# Key names — matches service-common ReferenceHierarchyKeys +BAGGAGE_KEY_TYPE = "ref.type" +BAGGAGE_KEY_ID = "ref.id" +BAGGAGE_KEY_VERSION = "ref.v" + + +@dataclass(frozen=True) +class ReferenceEntry: + """A single node in the reference hierarchy call chain.""" + + service_type: str + reference_id: str # UUID string + version: Optional[str] = None + + +class ReferenceContext: + """Immutable, copy-on-write ordered list of reference entries. + + Outermost caller first, current service appended last. + Each mutating call returns a new instance — the original is never + modified, preventing sibling spans from sharing context. + + Usage:: + + ctx = ReferenceContext.Empty + ctx = ctx.add("maestro", process_id, "2.1.0") + ctx = ctx.add("agent", agent_id) + token = ReferenceContextAccessor.set(ctx) + try: + ... + finally: + ReferenceContextAccessor.reset(token) + """ + + Empty: ClassVar["ReferenceContext"] + __slots__ = ("_entries",) + + def __init__(self, entries: Tuple[ReferenceEntry, ...] = ()) -> None: + self._entries: Tuple[ReferenceEntry, ...] = entries + + @property + def entries(self) -> Tuple[ReferenceEntry, ...]: + return self._entries + + def __len__(self) -> int: + return len(self._entries) + + def __iter__(self) -> Iterator[ReferenceEntry]: + return iter(self._entries) + + def __bool__(self) -> bool: + return len(self._entries) > 0 + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ReferenceContext): + return NotImplemented + return self._entries == other._entries + + def __hash__(self) -> int: + return hash(self._entries) + + def add( + self, + service_type: str, + reference_id: str | uuid.UUID, + version: Optional[str] = None, + ) -> "ReferenceContext": + """Returns a new context with this entry appended (copy-on-write). + + Args: + service_type: Identifier for the calling service (e.g. ``"agent"``, + ``"maestro"``). + reference_id: UUID of the referenced entity (UUID object or string). + version: Optional version string. + + Returns: + A new :class:`ReferenceContext` with the entry appended. + """ + if not service_type or not service_type.strip(): + raise ValueError("service_type must be a non-empty string.") + if isinstance(reference_id, uuid.UUID): + id_str = str(reference_id) + elif isinstance(reference_id, str): + try: + id_str = str(uuid.UUID(reference_id)) + except ValueError as exc: + raise ValueError( + f"reference_id {reference_id!r} is not a valid UUID." + ) from exc + else: + raise TypeError("reference_id must be a UUID or string.") + entry = ReferenceEntry( + service_type=service_type, + reference_id=id_str, + version=version if version and version.strip() else None, + ) + return ReferenceContext(self._entries + (entry,)) + + def to_wire_list(self) -> List[Dict[str, str]]: + """Serialize to the ``referenceHierarchy`` wire format. + + Returns: + A list of dicts suitable for JSON serialization as + ``Context.referenceHierarchy`` in the span payload. + """ + result: List[Dict[str, str]] = [] + for e in self._entries: + item: Dict[str, str] = { + "serviceType": e.service_type, + "referenceId": e.reference_id, + } + if e.version: + item["version"] = e.version + result.append(item) + return result + + @staticmethod + def from_baggage_header(header_value: Optional[str]) -> "ReferenceContext": + """Parse ``x-uipath-tracebaggage`` header value into a ReferenceContext. + + Only entries that carry the ``ref.*`` shape (type + valid UUID id) are + included. Malformed or plain-KV entries are silently skipped so a bad + header from an upstream service cannot crash this one. + + Args: + header_value: Raw header string, e.g. + ``"ref.type=agent;ref.id=;ref.v=1.0,ref.type=maestro;ref.id="`` + + Returns: + Parsed :class:`ReferenceContext`, or :attr:`ReferenceContext.Empty` + if the header is absent, empty, or contains no valid ref entries. + """ + if not header_value or not header_value.strip(): + return ReferenceContext.Empty + + entries: List[ReferenceEntry] = [] + for raw_entry in header_value.split(","): + entry_text = raw_entry.strip() + if not entry_text: + continue + props: dict[str, str] = {} + for raw_pair in entry_text.split(";"): + pair_text = raw_pair.strip() + eq = pair_text.find("=") + if eq <= 0 or eq >= len(pair_text) - 1: + continue + key = pair_text[:eq].strip() + value = pair_text[eq + 1 :].strip() + if key and value: + props[key] = value + + type_v = props.get(BAGGAGE_KEY_TYPE) + id_v = props.get(BAGGAGE_KEY_ID) + if not type_v or not id_v: + continue + try: + parsed_uuid = uuid.UUID(id_v) + except (ValueError, AttributeError): + continue + entries.append( + ReferenceEntry( + service_type=type_v, + reference_id=str(parsed_uuid), + version=props.get(BAGGAGE_KEY_VERSION) or None, + ) + ) + + if not entries: + return ReferenceContext.Empty + return ReferenceContext(tuple(entries)) + + def to_baggage_header_value(self) -> str: + """Serialize to ``x-uipath-tracebaggage`` header value. + + Returns: + Comma-separated entries; each is a semicolon-separated list of + ``key=value`` pairs. Empty context returns ``""``. + """ + if not self._entries: + return "" + parts: List[str] = [] + for e in self._entries: + kv = ( + f"{BAGGAGE_KEY_TYPE}={e.service_type};{BAGGAGE_KEY_ID}={e.reference_id}" + ) + if e.version: + kv += f";{BAGGAGE_KEY_VERSION}={e.version}" + parts.append(kv) + return ",".join(parts) + + +# Assigned after class body so ReferenceContext is fully bound. +ReferenceContext.Empty = ReferenceContext() + + +class ReferenceContextAccessor: + """Ambient accessor for the current :class:`ReferenceContext`. + + Backed by :mod:`contextvars` so the value propagates across ``await`` + boundaries without being threaded through every call signature. + + Usage:: + + token = ReferenceContextAccessor.set(ctx) + try: + ... # code here sees ReferenceContextAccessor.get() == ctx + finally: + ReferenceContextAccessor.reset(token) + """ + + _current: contextvars.ContextVar[Optional[ReferenceContext]] = ( + contextvars.ContextVar("uipath_reference_context", default=None) + ) + + @classmethod + def get(cls) -> Optional[ReferenceContext]: + """Return the current ambient context, or ``None`` if not set.""" + return cls._current.get() + + @classmethod + def set( + cls, value: Optional[ReferenceContext] + ) -> contextvars.Token[Optional[ReferenceContext]]: + """Set the ambient context. Returns a token for restoration. + + Pass the token to :meth:`reset` in a ``finally`` block. + """ + return cls._current.set(value) + + @classmethod + def reset(cls, token: contextvars.Token[Optional[ReferenceContext]]) -> None: + """Restore the ambient context to its prior value.""" + cls._current.reset(token) diff --git a/packages/uipath-platform/src/uipath/platform/common/_resource_identifier.py b/packages/uipath-platform/src/uipath/platform/common/_resource_identifier.py new file mode 100644 index 000000000..3f90ae774 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/common/_resource_identifier.py @@ -0,0 +1,12 @@ +"""Shared retrieve-identifier resolution for name/slug resources (MCP servers and Remote A2A agents).""" + + +def resolve_retrieve_identifier(name: str | None, slug: str | None) -> str: + """Resolve a retrieve identifier, preferring the display name over the legacy slug.""" + if name is not None and slug is not None: + raise ValueError("Specify either 'name' or 'slug', not both.") + if name is not None: + return name + if slug is not None: + return slug + raise TypeError("Either 'name' or 'slug' must be provided.") diff --git a/packages/uipath-platform/src/uipath/platform/common/_service_url_overrides.py b/packages/uipath-platform/src/uipath/platform/common/_service_url_overrides.py index f0a56318d..816285506 100644 --- a/packages/uipath-platform/src/uipath/platform/common/_service_url_overrides.py +++ b/packages/uipath-platform/src/uipath/platform/common/_service_url_overrides.py @@ -14,6 +14,11 @@ import os +from uipath.platform.constants import ( + HEADER_INTERNAL_ACCOUNT_ID, + HEADER_INTERNAL_TENANT_ID, +) + from ._config import UiPathConfig @@ -57,8 +62,8 @@ def inject_routing_headers(headers: dict[str, str]) -> None: """ tenant_id = UiPathConfig.tenant_id if tenant_id: - headers["X-UiPath-Internal-TenantId"] = tenant_id + headers[HEADER_INTERNAL_TENANT_ID] = tenant_id organization_id = UiPathConfig.organization_id if organization_id: - headers["X-UiPath-Internal-AccountId"] = organization_id + headers[HEADER_INTERNAL_ACCOUNT_ID] = organization_id diff --git a/packages/uipath-platform/src/uipath/platform/common/_span_utils.py b/packages/uipath-platform/src/uipath/platform/common/_span_utils.py index cd7e15e23..111dd59ec 100644 --- a/packages/uipath-platform/src/uipath/platform/common/_span_utils.py +++ b/packages/uipath-platform/src/uipath/platform/common/_span_utils.py @@ -2,21 +2,199 @@ import json import logging import os +import uuid from dataclasses import dataclass, field from datetime import datetime -from enum import IntEnum +from enum import IntEnum, StrEnum +from functools import lru_cache from os import environ as env -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, TypeVar -from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry import context as context_api +from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor from opentelemetry.trace import StatusCode from pydantic import BaseModel, ConfigDict, Field from uipath.core.serialization import serialize_json +from uipath.platform.constants import ( + ENV_FOLDER_KEY, + ENV_JOB_KEY, + ENV_ORGANIZATION_ID, + ENV_PROCESS_KEY, + ENV_TENANT_ID, + ENV_UIPATH_PROCESS_UUID, + ENV_UIPATH_PROCESS_VERSION, + ENV_UIPATH_TRACE_ID, +) + +from ._reference_context import ReferenceContextAccessor + + +def _inject_reference_hierarchy(span: Span) -> None: + ref_ctx = ReferenceContextAccessor.get() + if ref_ctx: + wire = ref_ctx.to_wire_list() + if wire: + span.set_attribute("uipath.reference_hierarchy", json.dumps(wire)) + + +class ReferenceHierarchySpanProcessor(SpanProcessor): + """Stamps uipath.reference_hierarchy on every span at creation time. + + Runs on_start in the span-creating thread so ContextVar values are live. + Register this before any processor that reads the attribute on on_start + (e.g. LiveTrackingSpanProcessor). + """ + + def on_start( + self, span: Span, parent_context: Optional[context_api.Context] = None + ) -> None: + _inject_reference_hierarchy(span) + + logger = logging.getLogger(__name__) -# SourceEnum.Robots = 4 (default for Python SDK / coded agents) -DEFAULT_SOURCE = 4 + +class SpanStatus(StrEnum): + UNSET = "Unset" + OK = "Ok" + ERROR = "Error" + RUNNING = "Running" + RESTRICTED = "Restricted" + CANCELLED = "Cancelled" + + +class SpanSource(StrEnum): + # Mirrors the server's SourceEnum + # (llm-observability: UiPath.LLMOps.DataAccess/Models/SourceEnum.cs). + # Member name = exact wire string (no naming policy on the v3 API). + # Keep complete: an unknown int is relabeled CodedAgents (see + # otel_span_to_uipath_span), and v3 rejects raw integers. + TESTING = "Testing" + AGENTS = "Agents" + PROCESS_ORCHESTRATION = "ProcessOrchestration" + API_WORKFLOWS = "ApiWorkflows" + ROBOTS = "Robots" + CONVERSATIONAL_AGENTS_SERVICE = "ConversationalAgentsService" + INTEGRATION_SERVICE_TRIGGER = "IntegrationServiceTrigger" + PLAYGROUND = "Playground" + GOVERNANCE = "Governance" + IXP_UNSTRUCTURED_AND_COMPLEX_DOCUMENTS = "IXPUnstructuredAndComplexDocuments" + CODED_AGENTS = "CodedAgents" + IXP_COMMUNICATIONS_MINING = "IXPCommunicationsMining" + ENTERPRISE_CONTEXT_SERVICE = "EnterpriseContextService" + MCP = "MCP" + A2A = "A2A" + SERVERLESS = "Serverless" + DOCUMENT_UNDERSTANDING = "DocumentUnderstanding" + + +class VerbosityLevel(StrEnum): + VERBOSE = "Verbose" + TRACE = "Trace" + INFORMATION = "Information" + WARNING = "Warning" + ERROR = "Error" + CRITICAL = "Critical" + OFF = "Off" + + +class ExecutionType(StrEnum): + DEBUG = "Debug" + RUNTIME = "Runtime" + + +# Int→StrEnum lookup tables for converting raw OTEL attribute integers +_EXECUTION_TYPE_BY_INT: dict[int, ExecutionType] = { + 0: ExecutionType.DEBUG, + 1: ExecutionType.RUNTIME, +} + +_VERBOSITY_LEVEL_BY_INT: dict[int, VerbosityLevel] = { + 0: VerbosityLevel.VERBOSE, + 1: VerbosityLevel.TRACE, + 2: VerbosityLevel.INFORMATION, + 3: VerbosityLevel.WARNING, + 4: VerbosityLevel.ERROR, + 5: VerbosityLevel.CRITICAL, + 6: VerbosityLevel.OFF, +} + +_SOURCE_BY_INT: dict[int, SpanSource] = { + 0: SpanSource.TESTING, + 1: SpanSource.AGENTS, + 2: SpanSource.PROCESS_ORCHESTRATION, + 3: SpanSource.API_WORKFLOWS, + 4: SpanSource.ROBOTS, + 5: SpanSource.CONVERSATIONAL_AGENTS_SERVICE, + 6: SpanSource.INTEGRATION_SERVICE_TRIGGER, + 7: SpanSource.PLAYGROUND, + 8: SpanSource.GOVERNANCE, + 9: SpanSource.IXP_UNSTRUCTURED_AND_COMPLEX_DOCUMENTS, + 10: SpanSource.CODED_AGENTS, + 11: SpanSource.IXP_COMMUNICATIONS_MINING, + 12: SpanSource.ENTERPRISE_CONTEXT_SERVICE, + 13: SpanSource.MCP, + 14: SpanSource.A2A, + 15: SpanSource.SERVERLESS, + 16: SpanSource.DOCUMENT_UNDERSTANDING, +} + +_IntEnumT = TypeVar("_IntEnumT") + + +def _enum_from_raw(table: dict[int, _IntEnumT], raw: Any) -> Optional[_IntEnumT]: + """Map a raw OTEL attribute to its StrEnum member, or None. + + Accepts a legacy int (looked up in ``table``) or the v3 string enum value + like ``"Off"`` (which also matches an already-typed member). ``bool`` is + rejected (``True == 1`` would match the value-1 member); unknown values + return None so callers can apply their own default. + """ + if isinstance(raw, bool): + return None + if isinstance(raw, int): + return table.get(raw) + if isinstance(raw, str): + for member in table.values(): + if member == raw: + return member + return None + return None + + +@lru_cache(maxsize=1) +def _read_config_id() -> str | None: + """Return a valid GUID ``id`` from ``uipath.json``, cached for the process lifetime.""" + from uipath.platform.common._config import UiPathConfig + + try: + config_file = json.loads(UiPathConfig.config_file_path.read_text()) + except (FileNotFoundError, json.JSONDecodeError): + return None + + project_id = config_file.get("id") + if not isinstance(project_id, str): + logger.warning("'id' field not present in uipath.json") + return None + + try: + uuid.UUID(project_id) + except ValueError: + logger.warning("Ignoring uipath.json 'id' %r: not a valid GUID.", project_id) + return None + + return project_id + + +def resolve_project_id() -> str | None: + """Resolve the project id. + + Prefers ``uipath.json#id``, falls back to env vars. + """ + from uipath.platform.common._config import UiPathConfig + + return _read_config_id() or UiPathConfig.agent_id or UiPathConfig.project_key class AttachmentProvider(IntEnum): @@ -59,35 +237,45 @@ class UiPathSpan: attributes: str | Dict[str, Any] # Support both str (legacy) and dict (optimized) parent_id: Optional[str] = None # 16-char hex (OTEL span ID format) start_time: str = field(default_factory=lambda: datetime.now().isoformat()) - end_time: str = field(default_factory=lambda: datetime.now().isoformat()) - status: int = 1 + # None means the span has not ended yet. Serialized as null — the LLMOps v3 + # ingest contract (SpanV3Req.EndTime) is nullable, and downstream consumers + # (traceview UI, Insights OTLP export) rely on "EndTime set" meaning "span + # ended": fabricating a value here makes in-progress snapshots look terminal. + end_time: Optional[str] = None + status: SpanStatus = SpanStatus.OK created_at: str = field(default_factory=lambda: datetime.now().isoformat() + "Z") updated_at: str = field(default_factory=lambda: datetime.now().isoformat() + "Z") + # Default to None (not "") when unset; to_dict() then omits these keys + # entirely. The v3 ingest endpoint binds them to Guid fields: "" crashes the + # serializer, and even null fails for the required OrganizationId/FolderKey. + # In the platform runtime these are always set to real GUIDs. organization_id: Optional[str] = field( - default_factory=lambda: env.get("UIPATH_ORGANIZATION_ID", "") + default_factory=lambda: env.get(ENV_ORGANIZATION_ID) or None ) tenant_id: Optional[str] = field( - default_factory=lambda: env.get("UIPATH_TENANT_ID", "") + default_factory=lambda: env.get(ENV_TENANT_ID) or None ) expiry_time_utc: Optional[str] = None folder_key: Optional[str] = field( - default_factory=lambda: env.get("UIPATH_FOLDER_KEY", "") + default_factory=lambda: env.get(ENV_FOLDER_KEY) or None ) - source: int = DEFAULT_SOURCE + source: SpanSource = SpanSource.CODED_AGENTS span_type: str = "Coded Agents" process_key: Optional[str] = field( - default_factory=lambda: env.get("UIPATH_PROCESS_UUID") + default_factory=lambda: env.get(ENV_UIPATH_PROCESS_UUID) ) reference_id: Optional[str] = field( default_factory=lambda: env.get("TRACE_REFERENCE_ID") ) - job_key: Optional[str] = field(default_factory=lambda: env.get("UIPATH_JOB_KEY")) + job_key: Optional[str] = field(default_factory=lambda: env.get(ENV_JOB_KEY)) # Top-level fields for internal tracing schema - execution_type: Optional[int] = None + execution_type: Optional[ExecutionType] = None agent_version: Optional[str] = None + verbosity_level: Optional[VerbosityLevel] = None attachments: Optional[List[SpanAttachment]] = None + context: Optional[Dict[str, Any]] = None def to_dict(self, serialize_attributes: bool = True) -> Dict[str, Any]: """Convert the Span to a dictionary suitable for JSON serialization. @@ -114,7 +302,7 @@ def to_dict(self, serialize_attributes: bool = True) -> Dict[str, Any]: for att in self.attachments ] - return { + result: Dict[str, Any] = { "Id": self.id, "TraceId": self.trace_id, "ParentId": self.parent_id, @@ -135,9 +323,21 @@ def to_dict(self, serialize_attributes: bool = True) -> Dict[str, Any]: "JobKey": self.job_key, "ReferenceId": self.reference_id, "ExecutionType": self.execution_type, - "AgentVersion": self.agent_version, + # v3 ingest (SpanV3Req) has no AgentVersion field; the agent version + # is carried by ReferenceVersion (pairs with ReferenceId above). + "ReferenceVersion": self.agent_version, "Attachments": attachments_out, } + # Omit Guid-typed keys when unset — v3 binds them to Guid columns and + # rejects null/"". When present (platform runtime) they hold real GUIDs. + for guid_key in ("OrganizationId", "TenantId", "FolderKey"): + if result[guid_key] is None: + del result[guid_key] + if self.verbosity_level is not None: + result["VerbosityLevel"] = self.verbosity_level + if self.context is not None: + result["Context"] = self.context + return result class _SpanUtils: @@ -193,7 +393,7 @@ def otel_span_to_uipath_span( span_id = format(span_context.span_id, "016x") # Override trace_id if custom or env var provided (supports both UUID and hex format) - trace_id_override = custom_trace_id or os.environ.get("UIPATH_TRACE_ID") + trace_id_override = custom_trace_id or os.environ.get(ENV_UIPATH_TRACE_ID) if trace_id_override: trace_id = _SpanUtils.normalize_trace_id(trace_id_override) @@ -213,10 +413,15 @@ def otel_span_to_uipath_span( # Only copy if we need to modify - we'll build attributes_dict lazily attributes_dict: dict[str, Any] = dict(otel_attrs) if otel_attrs else {} + # Pull the reference hierarchy stamped by the span-start hook (runs in the + # correct thread/context; BatchSpanProcessor exports in a background thread + # where ContextVar values are not available). + ref_hierarchy_json = attributes_dict.pop("uipath.reference_hierarchy", None) + # Map status - status = 1 # Default to OK + status = SpanStatus.OK if otel_span.status.status_code == StatusCode.ERROR: - status = 2 # Error + status = SpanStatus.ERROR attributes_dict["error"] = otel_span.status.description # Process inputs - avoid redundant parsing if already parsed @@ -267,11 +472,13 @@ def otel_span_to_uipath_span( ] attributes_dict["links"] = links_list + if agent_id := resolve_project_id(): + attributes_dict["agentId"] = agent_id + # Add process context attributes from environment variables for env_key, attr_key in ( - ("PROJECT_KEY", "agentId"), - ("UIPATH_PROCESS_KEY", "agentName"), - ("UIPATH_PROCESS_VERSION", "agentVersion"), + (ENV_PROCESS_KEY, "agentName"), + (ENV_UIPATH_PROCESS_VERSION, "agentVersion"), ): value = env.get(env_key) if value: @@ -280,14 +487,34 @@ def otel_span_to_uipath_span( span_type_value = attributes_dict.get("span_type", "OpenTelemetry") span_type = str(span_type_value) - # Top-level fields for internal tracing schema - execution_type = attributes_dict.get("executionType") + # Top-level fields for internal tracing schema. The enum lookups go + # through _enum_from_raw, which accepts both the legacy integer wire form + # and the v3 string-enum value (and rejects bools) identically. + execution_type = _enum_from_raw( + _EXECUTION_TYPE_BY_INT, attributes_dict.get("executionType") + ) agent_version = attributes_dict.get("agentVersion") - reference_id = attributes_dict.get("referenceId") + reference_id = attributes_dict.get("agentId") or attributes_dict.get( + "referenceId" + ) + verbosity_level = _enum_from_raw( + _VERBOSITY_LEVEL_BY_INT, attributes_dict.get("verbosityLevel") + ) - # Source: override via uipath.source attribute, else DEFAULT_SOURCE - uipath_source = attributes_dict.get("uipath.source") - source = uipath_source if isinstance(uipath_source, int) else DEFAULT_SOURCE + # Source: override via uipath.source attribute, else CodedAgents. + # A real int that isn't a known source is relabeled CodedAgents but + # logged — v3 ingest rejects raw integers, so it can't be forwarded. + uipath_source_raw = attributes_dict.get("uipath.source") + source = _enum_from_raw(_SOURCE_BY_INT, uipath_source_raw) + if source is None: + if isinstance(uipath_source_raw, int) and not isinstance( + uipath_source_raw, bool + ): + logger.warning( + "Unknown uipath.source int %s; defaulting to CodedAgents", + uipath_source_raw, + ) + source = SpanSource.CODED_AGENTS attachments = None attachments_data = attributes_dict.get("attachments") @@ -307,18 +534,26 @@ def otel_span_to_uipath_span( except Exception as e: logger.warning(f"Error processing attachments: {e}") + context: Optional[Dict[str, Any]] = None + if ref_hierarchy_json: + try: + context = {"referenceHierarchy": json.loads(ref_hierarchy_json)} + except (json.JSONDecodeError, TypeError): + pass + # Create UiPathSpan from OpenTelemetry span start_time = datetime.fromtimestamp( (otel_span.start_time or 0) / 1e9 ).isoformat() + # A live (not-yet-ended) OTel span has end_time None — preserve that rather + # than stamping now(): a fabricated EndTime makes in-progress upserts + # (upsert_span RUNNING → OK lifecycle) indistinguishable from ended spans. end_time_str = None if otel_span.end_time is not None: end_time_str = datetime.fromtimestamp( (otel_span.end_time or 0) / 1e9 ).isoformat() - else: - end_time_str = datetime.now().isoformat() return UiPathSpan( id=span_id, @@ -334,9 +569,11 @@ def otel_span_to_uipath_span( span_type=span_type, execution_type=execution_type, agent_version=agent_version, + verbosity_level=verbosity_level, reference_id=reference_id, source=source, attachments=attachments, + context=context, ) @staticmethod diff --git a/packages/uipath-platform/src/uipath/platform/common/_user_agent.py b/packages/uipath-platform/src/uipath/platform/common/_user_agent.py index dcf28d3e0..1083faa45 100644 --- a/packages/uipath-platform/src/uipath/platform/common/_user_agent.py +++ b/packages/uipath-platform/src/uipath/platform/common/_user_agent.py @@ -1,6 +1,6 @@ import importlib -from .constants import HEADER_USER_AGENT +from uipath.platform.constants import HEADER_USER_AGENT def user_agent_value(specific_component: str) -> str: diff --git a/packages/uipath-platform/src/uipath/platform/common/auth.py b/packages/uipath-platform/src/uipath/platform/common/auth.py index 885a0ef1b..a8f87d3e4 100644 --- a/packages/uipath-platform/src/uipath/platform/common/auth.py +++ b/packages/uipath-platform/src/uipath/platform/common/auth.py @@ -5,7 +5,7 @@ from pydantic import BaseModel -from uipath.platform.common.constants import ( +from uipath.platform.constants import ( ENV_BASE_URL, ENV_UIPATH_ACCESS_TOKEN, ENV_UNATTENDED_USER_ACCESS_TOKEN, diff --git a/packages/uipath-platform/src/uipath/platform/common/constants.py b/packages/uipath-platform/src/uipath/platform/common/constants.py index 6184e844d..5a12ee4b0 100644 --- a/packages/uipath-platform/src/uipath/platform/common/constants.py +++ b/packages/uipath-platform/src/uipath/platform/common/constants.py @@ -1,89 +1,16 @@ -"""Constants.""" +"""Deprecated alias for ``uipath.platform.constants``. -# Environment variables -DOTENV_FILE = ".env" -ENV_BASE_URL = "UIPATH_URL" -ENV_EVAL_BACKEND_URL = "UIPATH_EVAL_BACKEND_URL" -ENV_UNATTENDED_USER_ACCESS_TOKEN = "UNATTENDED_USER_ACCESS_TOKEN" -ENV_UIPATH_ACCESS_TOKEN = "UIPATH_ACCESS_TOKEN" -ENV_FOLDER_KEY = "UIPATH_FOLDER_KEY" -ENV_FOLDER_PATH = "UIPATH_FOLDER_PATH" -ENV_JOB_KEY = "UIPATH_JOB_KEY" -ENV_JOB_ID = "UIPATH_JOB_ID" -ENV_ROBOT_KEY = "UIPATH_ROBOT_KEY" -ENV_TENANT_ID = "UIPATH_TENANT_ID" -ENV_TENANT_NAME = "UIPATH_TENANT_NAME" -ENV_ORGANIZATION_ID = "UIPATH_ORGANIZATION_ID" -ENV_TELEMETRY_ENABLED = "UIPATH_TELEMETRY_ENABLED" -ENV_TRACING_ENABLED = "UIPATH_TRACING_ENABLED" -ENV_UIPATH_PROJECT_ID = "UIPATH_PROJECT_ID" -ENV_PROJECT_KEY = "PROJECT_KEY" -ENV_PROCESS_KEY = "UIPATH_PROCESS_KEY" -ENV_UIPATH_PROCESS_UUID = "UIPATH_PROCESS_UUID" -ENV_UIPATH_TRACE_ID = "UIPATH_TRACE_ID" -ENV_UIPATH_PROCESS_VERSION = "UIPATH_PROCESS_VERSION" -ENV_UIPATH_CONFIG_PATH = "UIPATH_CONFIG_PATH" +This module is kept as a backward-compatibility shim so existing imports keep +working. New code should import from ``uipath.platform.constants``. +""" -# Headers -HEADER_FOLDER_KEY = "x-uipath-folderkey" -HEADER_FOLDER_PATH = "x-uipath-folderpath" -HEADER_FOLDER_PATH_ENCODED = "x-uipath-folderpath-encoded" -HEADER_USER_AGENT = "x-uipath-user-agent" -HEADER_TENANT_ID = "x-uipath-tenantid" -HEADER_INTERNAL_TENANT_ID = "x-uipath-internal-tenantid" -HEADER_INTERNAL_ACCOUNT_ID = "x-uipath-internal-accountid" -HEADER_JOB_KEY = "x-uipath-jobkey" -HEADER_PROCESS_KEY = "x-uipath-processkey" -HEADER_TRACE_ID = "x-uipath-traceid" -HEADER_AGENTHUB_CONFIG = "x-uipath-agenthub-config" -HEADER_LLMGATEWAY_BYO_CONNECTION_ID = "x-uipath-llmgateway-byoisconnectionid" -HEADER_SW_LOCK_KEY = "x-uipath-sw-lockkey" -HEADER_LICENSING_CONTEXT = "x-uipath-licensing-context" +import warnings as _warnings -# Data sources (request types) -ORCHESTRATOR_STORAGE_BUCKET_DATA_SOURCE_REQUEST = ( - "#UiPath.Vdbs.Domain.Api.V20Models.StorageBucketDataSourceRequest" -) -CONFLUENCE_DATA_SOURCE_REQUEST = ( - "#UiPath.Vdbs.Domain.Api.V20Models.ConfluenceDataSourceRequest" -) -DROPBOX_DATA_SOURCE_REQUEST = ( - "#UiPath.Vdbs.Domain.Api.V20Models.DropboxDataSourceRequest" -) -GOOGLE_DRIVE_DATA_SOURCE_REQUEST = ( - "#UiPath.Vdbs.Domain.Api.V20Models.GoogleDriveDataSourceRequest" -) -ONEDRIVE_DATA_SOURCE_REQUEST = ( - "#UiPath.Vdbs.Domain.Api.V20Models.OneDriveDataSourceRequest" -) +from uipath.platform.constants import * # noqa: F401,F403 -# Data sources -ORCHESTRATOR_STORAGE_BUCKET_DATA_SOURCE = ( - "#UiPath.Vdbs.Domain.Api.V20Models.StorageBucketDataSource" +_warnings.warn( + "uipath.platform.common.constants is deprecated and will be removed in a " + "future release; import from uipath.platform.constants instead.", + FutureWarning, + stacklevel=2, ) -CONFLUENCE_DATA_SOURCE = "#UiPath.Vdbs.Domain.Api.V20Models.ConfluenceDataSource" -DROPBOX_DATA_SOURCE = "#UiPath.Vdbs.Domain.Api.V20Models.DropboxDataSource" -GOOGLE_DRIVE_DATA_SOURCE = "#UiPath.Vdbs.Domain.Api.V20Models.GoogleDriveDataSource" -ONEDRIVE_DATA_SOURCE = "#UiPath.Vdbs.Domain.Api.V20Models.OneDriveDataSource" - - -# Local storage -TEMP_ATTACHMENTS_FOLDER = "uipath_attachments" - -# LLM models -COMMUNITY_agents_SUFFIX = "-community-agents" - -# File names -PYTHON_CONFIGURATION_FILE = "pyproject.toml" -UIPATH_CONFIG_FILE = "uipath.json" -UIPATH_BINDINGS_FILE = "bindings.json" -ENTRY_POINTS_FILE = "entry-points.json" -STUDIO_METADATA_FILE = "studio_metadata.json" -UIPROJ_FILE = "project.uiproj" - - -# Folder names -LEGACY_EVAL_FOLDER = "evals" -EVALS_FOLDER = "evaluations" -# Evaluators -CUSTOM_EVALUATOR_PREFIX = "file://" diff --git a/packages/uipath-platform/src/uipath/platform/common/interrupt_models.py b/packages/uipath-platform/src/uipath/platform/common/interrupt_models.py index 100b601bd..ee9104c54 100644 --- a/packages/uipath-platform/src/uipath/platform/common/interrupt_models.py +++ b/packages/uipath-platform/src/uipath/platform/common/interrupt_models.py @@ -1,8 +1,9 @@ """Models for interrupt operations in UiPath platform.""" +from datetime import datetime, timezone from typing import Annotated, Any -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from uipath.platform.context_grounding.context_grounding_index import ( ContextGroundingIndex, @@ -259,3 +260,39 @@ class WaitDocumentExtractionValidation(BaseModel): extraction_validation: StartExtractionValidationResponse task_url: str | None = None + + +class WaitIntegrationEvent(BaseModel): + """Model representing a wait on an Integration Services event. + + Used to suspend a job until a remote event (e.g. Slack message, Teams reply) + is delivered by Integration Services. The SDK resolves `connection_name` + (scoped to `connection_folder_path` when provided) to the underlying + connection id and generates a fresh `inbox_id` when the trigger is created; + the rest of the fields describe which remote event to subscribe to via + the Connections service. + """ + + connector: str + connection_name: str + connection_folder_path: str | None = None + operation: str + object_name: str + filter_expression: str | None = None + parameters: dict[str, str] | None = None + + +class WaitUntil(BaseModel): + """Model representing a wait until an absolute point in time.""" + + resume_time: datetime = Field(alias="resumeTime") + + model_config = ConfigDict(validate_by_name=True) + + @field_validator("resume_time") + @classmethod + def validate_resume_time(cls, value: datetime) -> datetime: + """Validate and normalize resume_time to a UTC instant.""" + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("resume_time must include timezone information") + return value.astimezone(timezone.utc) diff --git a/packages/uipath-platform/src/uipath/platform/common/timeout.py b/packages/uipath-platform/src/uipath/platform/common/timeout.py new file mode 100644 index 000000000..20b79873c --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/common/timeout.py @@ -0,0 +1,57 @@ +"""Helpers for resume values produced by timeout triggers.""" + +from collections.abc import Mapping +from typing import Any, TypeVar + +from pydantic import ValidationError +from uipath.core.triggers import ( + UIPATH_METADATA_KEY, + UiPathResumeMetadata, + UiPathResumeTriggerType, +) + +T = TypeVar("T") + + +class UiPathTimeoutError(TimeoutError): + """Raised when a resume value came from a UiPath timeout trigger.""" + + def __init__(self, value: Any): + """Create a timeout error that keeps the original resume value.""" + super().__init__("UiPath interrupt timed out.") + self.value = value + + +def is_timeout(value: Any) -> bool: + """Return True when a resume value came from a UiPath timeout trigger.""" + metadata = get_resume_metadata(value) + return ( + metadata is not None and metadata.trigger_type == UiPathResumeTriggerType.TIMER + ) + + +def assert_no_timeout(value: T) -> T: + """Raise UiPathTimeoutError if a resume value came from a timeout trigger.""" + if is_timeout(value): + raise UiPathTimeoutError(value) + return value + + +def get_resume_metadata(value: Any) -> UiPathResumeMetadata | None: + """Return UiPath resume metadata when present on a resume value.""" + metadata = _metadata(value) + if metadata is None: + return None + + try: + return UiPathResumeMetadata.model_validate(metadata) + except ValidationError: + return None + + +def _metadata(value: Any) -> Mapping[str, Any] | None: + if not isinstance(value, Mapping): + return None + + metadata = value.get(UIPATH_METADATA_KEY) + return metadata if isinstance(metadata, Mapping) else None diff --git a/packages/uipath-platform/src/uipath/platform/connections/_connections_service.py b/packages/uipath-platform/src/uipath/platform/connections/_connections_service.py index af3bb4f78..b7c1e9444 100644 --- a/packages/uipath-platform/src/uipath/platform/connections/_connections_service.py +++ b/packages/uipath-platform/src/uipath/platform/connections/_connections_service.py @@ -8,7 +8,7 @@ from ..common._base_service import BaseService from ..common._bindings import resource_override -from ..common._config import UiPathApiConfig +from ..common._config import UiPathApiConfig, UiPathConfig from ..common._execution_context import UiPathExecutionContext from ..common._folder_context import header_folder from ..common._models import Endpoint, RequestSpec @@ -24,6 +24,13 @@ logger: logging.Logger = logging.getLogger("uipath") +HEADER_ORIGINATOR = "x-uipath-originator" +HEADER_SOURCE = "x-uipath-source" +# Sent on outbound Integration Service activity invocations so GenAI activities +# can be stitched back to the parent job for licensing attribution. +HEADER_ACTIVITY_JOB_ID = "x-uipath-job-id" +_ORIGINATOR_VALUE = "uipath-python" + class ConnectionsService(BaseService): """Service for managing UiPath external service connections. @@ -768,11 +775,13 @@ def _build_activity_request_spec( # header parameter handling headers = { - "x-uipath-originator": "uipath-python", - "x-uipath-source": "uipath-python", + HEADER_ORIGINATOR: _ORIGINATOR_VALUE, + HEADER_SOURCE: _ORIGINATOR_VALUE, **header_folder(folder_key, None), **header_params, } + if job_key := UiPathConfig.job_key: + headers[HEADER_ACTIVITY_JOB_ID] = job_key # body and files handling json_data: Dict[str, Any] | None = None @@ -788,12 +797,22 @@ def _build_activity_request_spec( # instead of making assumptions on whether or not it's present, we'll handle it defensively if key == json_section: continue - # files not supported yet supported so this will likely not work - files[key] = ( - key, - val, - None, - ) # probably needs to extract content type from val since IS metadata doesn't provide it + if isinstance(val, tuple): + # Caller supplied httpx's (filename, content[, content_type]) + # shape — pass through verbatim. This is the recommended path + # for file uploads so the multipart Content-Disposition gets + # the real filename instead of the form-field name. + files[key] = val + elif isinstance(val, (bytes, bytearray)) or hasattr(val, "read"): + # Raw file content with no filename — fall back to the + # form-field name (legacy behaviour). Backwards compatible + # with callers that still pass bytes directly. + files[key] = (key, val, "application/octet-stream") + else: + # Scalar (string/number/etc.) — send as a plain multipart + # form field, not a file part. The (None, value) shape tells + # httpx to omit `filename=...` from the Content-Disposition. + files[key] = (None, str(val)) files[json_section] = ( "", diff --git a/packages/uipath-platform/src/uipath/platform/constants/__init__.py b/packages/uipath-platform/src/uipath/platform/constants/__init__.py new file mode 100644 index 000000000..fc28a10c8 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/constants/__init__.py @@ -0,0 +1,106 @@ +"""Constants. + +Single source of truth for constants shared across the UiPath SDK packages. +Lives directly under ``uipath.platform`` (sibling to ``common``) so it can be +imported cheaply — ``uipath.platform``'s ``__init__`` resolves its heavy +exports lazily, so importing this module does not pull in the service layer. + +``uipath.platform.common.constants`` and ``uipath._utils.constants`` re-export +from here. +""" + +# Environment variables +DOTENV_FILE = ".env" +ENV_BASE_URL = "UIPATH_URL" +ENV_EVAL_BACKEND_URL = "UIPATH_EVAL_BACKEND_URL" +ENV_UNATTENDED_USER_ACCESS_TOKEN = "UNATTENDED_USER_ACCESS_TOKEN" +ENV_UIPATH_ACCESS_TOKEN = "UIPATH_ACCESS_TOKEN" +ENV_FOLDER_KEY = "UIPATH_FOLDER_KEY" +ENV_FOLDER_PATH = "UIPATH_FOLDER_PATH" +ENV_JOB_KEY = "UIPATH_JOB_KEY" +ENV_JOB_ID = "UIPATH_JOB_ID" +ENV_ROBOT_KEY = "UIPATH_ROBOT_KEY" +ENV_TENANT_ID = "UIPATH_TENANT_ID" +ENV_TENANT_NAME = "UIPATH_TENANT_NAME" +ENV_ORGANIZATION_ID = "UIPATH_ORGANIZATION_ID" +ENV_TELEMETRY_ENABLED = "UIPATH_TELEMETRY_ENABLED" +ENV_TRACING_ENABLED = "UIPATH_TRACING_ENABLED" +ENV_UIPATH_PROJECT_ID = "UIPATH_PROJECT_ID" +ENV_UIPATH_AGENT_ID = "UIPATH_AGENT_ID" +ENV_UIPATH_CLOUD_USER_ID = "UIPATH_CLOUD_USER_ID" +ENV_UIPATH_PROJECT_FILES_SOURCE = "UIPATH_PROJECT_FILES_SOURCE" +ENV_PROJECT_KEY = "PROJECT_KEY" +ENV_PROCESS_KEY = "UIPATH_PROCESS_KEY" +ENV_UIPATH_PROCESS_UUID = "UIPATH_PROCESS_UUID" +ENV_UIPATH_TRACE_ID = "UIPATH_TRACE_ID" +ENV_UIPATH_PROCESS_VERSION = "UIPATH_PROCESS_VERSION" +ENV_UIPATH_CONFIG_PATH = "UIPATH_CONFIG_PATH" + +# Headers +HEADER_FOLDER_KEY = "x-uipath-folderkey" +HEADER_FOLDER_PATH = "x-uipath-folderpath" +HEADER_FOLDER_PATH_ENCODED = "x-uipath-folderpath-encoded" +HEADER_USER_AGENT = "x-uipath-user-agent" +HEADER_SOURCE = "x-uipath-source" +HEADER_TENANT_ID = "x-uipath-tenantid" +HEADER_INTERNAL_TENANT_ID = "x-uipath-internal-tenantid" +HEADER_INTERNAL_ACCOUNT_ID = "x-uipath-internal-accountid" +HEADER_JOB_KEY = "x-uipath-jobkey" +HEADER_PROCESS_KEY = "x-uipath-processkey" +HEADER_TRACE_ID = "x-uipath-traceid" +HEADER_AGENTHUB_CONFIG = "x-uipath-agenthub-config" +HEADER_GUARDRAILS_SOURCE = "x-uipath-guardrails-source" +HEADER_LLMGATEWAY_BYO_CONNECTION_ID = "x-uipath-llmgateway-byoisconnectionid" +HEADER_SW_LOCK_KEY = "x-uipath-sw-lockkey" +HEADER_LICENSING_CONTEXT = "x-uipath-licensing-context" + +# Data sources (request types) +ORCHESTRATOR_STORAGE_BUCKET_DATA_SOURCE_REQUEST = ( + "#UiPath.Vdbs.Domain.Api.V20Models.StorageBucketDataSourceRequest" +) +CONFLUENCE_DATA_SOURCE_REQUEST = ( + "#UiPath.Vdbs.Domain.Api.V20Models.ConfluenceDataSourceRequest" +) +DROPBOX_DATA_SOURCE_REQUEST = ( + "#UiPath.Vdbs.Domain.Api.V20Models.DropboxDataSourceRequest" +) +GOOGLE_DRIVE_DATA_SOURCE_REQUEST = ( + "#UiPath.Vdbs.Domain.Api.V20Models.GoogleDriveDataSourceRequest" +) +ONEDRIVE_DATA_SOURCE_REQUEST = ( + "#UiPath.Vdbs.Domain.Api.V20Models.OneDriveDataSourceRequest" +) + +# Data sources +ORCHESTRATOR_STORAGE_BUCKET_DATA_SOURCE = ( + "#UiPath.Vdbs.Domain.Api.V20Models.StorageBucketDataSource" +) +CONFLUENCE_DATA_SOURCE = "#UiPath.Vdbs.Domain.Api.V20Models.ConfluenceDataSource" +DROPBOX_DATA_SOURCE = "#UiPath.Vdbs.Domain.Api.V20Models.DropboxDataSource" +GOOGLE_DRIVE_DATA_SOURCE = "#UiPath.Vdbs.Domain.Api.V20Models.GoogleDriveDataSource" +ONEDRIVE_DATA_SOURCE = "#UiPath.Vdbs.Domain.Api.V20Models.OneDriveDataSource" +LLMV3Mini_REQUEST = "#UiPath.Vdbs.Domain.Api.V20Models.LLMV3MiniPreProcessingRequest" +LLMV4_REQUEST = "#UiPath.Vdbs.Domain.Api.V20Models.LLMV4PreProcessingRequest" +NativeV1_REQUEST = "#UiPath.Vdbs.Domain.Api.V20Models.NativeV1PreProcessingRequest" + + +# Local storage +TEMP_ATTACHMENTS_FOLDER = "uipath_attachments" + +# LLM models +COMMUNITY_agents_SUFFIX = "-community-agents" + +# File names +PYTHON_CONFIGURATION_FILE = "pyproject.toml" +UIPATH_CONFIG_FILE = "uipath.json" +UIPATH_BINDINGS_FILE = "bindings.json" +ENTRY_POINTS_FILE = "entry-points.json" +STUDIO_METADATA_FILE = "studio_metadata.json" +UIPROJ_FILE = "project.uiproj" + + +# Folder names +LEGACY_EVAL_FOLDER = "evals" +EVALS_FOLDER = "evaluations" +# Evaluators +CUSTOM_EVALUATOR_PREFIX = "file://" diff --git a/packages/uipath-platform/src/uipath/platform/context_grounding/_context_grounding_service.py b/packages/uipath-platform/src/uipath/platform/context_grounding/_context_grounding_service.py index 75e525d3b..fc47d7b8b 100644 --- a/packages/uipath-platform/src/uipath/platform/context_grounding/_context_grounding_service.py +++ b/packages/uipath-platform/src/uipath/platform/context_grounding/_context_grounding_service.py @@ -6,18 +6,22 @@ from typing_extensions import deprecated from uipath.core.tracing import traced +from uipath.platform.constants import ( + ORCHESTRATOR_STORAGE_BUCKET_DATA_SOURCE, +) + from ..common._base_service import BaseService from ..common._bindings import resource_override from ..common._config import UiPathApiConfig from ..common._execution_context import UiPathExecutionContext from ..common._folder_context import FolderContext, header_folder from ..common._http_config import get_httpx_client_kwargs +from ..common._job_context import header_job_key from ..common._models import Endpoint, RequestSpec -from ..common.constants import ( - ORCHESTRATOR_STORAGE_BUCKET_DATA_SOURCE, -) from ..errors import ( + BatchTransformFailedException, BatchTransformNotCompleteException, + ContextGroundingIndexNotFoundError, IngestionInProgressException, UnsupportedDataSourceException, ) @@ -219,6 +223,7 @@ def retrieve_across_folders( spec.method, spec.endpoint, params=spec.params, + headers=spec.headers, ).json() return [ @@ -248,6 +253,45 @@ async def retrieve_across_folders_async( spec.method, spec.endpoint, params=spec.params, + headers=spec.headers, + ) + ).json() + + return [ + ContextGroundingIndex.model_validate(item) for item in response["value"] + ] + + @traced(name="contextgrounding_retrieve_system_indexes", run_type="uipath") + def _retrieve_system_indexes( + self, + name: Optional[str] = None, + ) -> List[ContextGroundingIndex]: + spec = self._retrieve_system_indexes_spec(name=name) + + response = self.request( + spec.method, + spec.endpoint, + params=spec.params, + headers=spec.headers, + ).json() + + return [ + ContextGroundingIndex.model_validate(item) for item in response["value"] + ] + + @traced(name="contextgrounding_retrieve_system_indexes", run_type="uipath") + async def _retrieve_system_indexes_async( + self, + name: Optional[str] = None, + ) -> List[ContextGroundingIndex]: + spec = self._retrieve_system_indexes_spec(name=name) + + response = ( + await self.request_async( + spec.method, + spec.endpoint, + params=spec.params, + headers=spec.headers, ) ).json() @@ -262,30 +306,38 @@ def retrieve( name: str, folder_key: Optional[str] = None, folder_path: Optional[str] = None, + include_system_indexes: bool = False, ) -> ContextGroundingIndex: """Retrieve context grounding index information by its name. If no folder_key or folder_path is provided and no folder context is - configured, falls back to searching across all folders. + configured, falls back to searching across all folders. When + ``include_system_indexes`` is True, an additional fallback against + system indexes is attempted before raising not-found. Args: name (str): The name of the context index to retrieve. folder_key (Optional[str]): The key of the folder where the index resides. folder_path (Optional[str]): The path of the folder where the index resides. + include_system_indexes (bool): If True, fall back to system indexes + when the index is not found in the per-folder or across-folders listings. + Defaults to False. Returns: ContextGroundingIndex: The index information, including its configuration and metadata if found. Raises: - Exception: If no index with the given name is found. + ContextGroundingIndexNotFoundError: If no index with the given name is found. """ resolved_folder_key = self._resolve_folder_key(folder_key, folder_path) if not resolved_folder_key: indexes = self.retrieve_across_folders(name=name) try: return next(index for index in indexes if index.name == name) - except StopIteration as e: - raise Exception("ContextGroundingIndex not found") from e + except StopIteration: + if include_system_indexes: + return self._retrieve_from_system_indexes(name) + raise ContextGroundingIndexNotFoundError(name) from None spec = self._retrieve_spec( name, @@ -304,8 +356,10 @@ def retrieve( for item in response["value"] if item["name"] == name ) - except StopIteration as e: - raise Exception("ContextGroundingIndex not found") from e + except StopIteration: + if include_system_indexes: + return self._retrieve_from_system_indexes(name) + raise ContextGroundingIndexNotFoundError(name) from None @resource_override(resource_type="index") @traced(name="contextgrounding_retrieve", run_type="uipath") @@ -314,30 +368,38 @@ async def retrieve_async( name: str, folder_key: Optional[str] = None, folder_path: Optional[str] = None, + include_system_indexes: bool = False, ) -> ContextGroundingIndex: """Asynchronously retrieve context grounding index information by its name. If no folder_key or folder_path is provided and no folder context is - configured, falls back to searching across all folders. + configured, falls back to searching across all folders. When + ``include_system_indexes`` is True, an additional fallback against + system indexes is attempted before raising not-found. Args: name (str): The name of the context index to retrieve. folder_key (Optional[str]): The key of the folder where the index resides. folder_path (Optional[str]): The path of the folder where the index resides. + include_system_indexes (bool): If True, fall back to system indexes when + the index is not found in the per-folder or across-folders listings. + Defaults to False. Returns: ContextGroundingIndex: The index information, including its configuration and metadata if found. Raises: - Exception: If no index with the given name is found. + ContextGroundingIndexNotFoundError: If no index with the given name is found. """ resolved_folder_key = self._resolve_folder_key(folder_key, folder_path) if not resolved_folder_key: indexes = await self.retrieve_across_folders_async(name=name) try: return next(index for index in indexes if index.name == name) - except StopIteration as e: - raise Exception("ContextGroundingIndex not found") from e + except StopIteration: + if include_system_indexes: + return await self._retrieve_from_system_indexes_async(name) + raise ContextGroundingIndexNotFoundError(name) from None spec = self._retrieve_spec( name, @@ -358,8 +420,26 @@ async def retrieve_async( for item in response["value"] if item["name"] == name ) - except StopIteration as e: - raise Exception("ContextGroundingIndex not found") from e + except StopIteration: + if include_system_indexes: + return await self._retrieve_from_system_indexes_async(name) + raise ContextGroundingIndexNotFoundError(name) from None + + def _retrieve_from_system_indexes(self, name: str) -> ContextGroundingIndex: + indexes = self._retrieve_system_indexes(name=name) + try: + return next(index for index in indexes if index.name == name) + except StopIteration: + raise ContextGroundingIndexNotFoundError(name) from None + + async def _retrieve_from_system_indexes_async( + self, name: str + ) -> ContextGroundingIndex: + indexes = await self._retrieve_system_indexes_async(name=name) + try: + return next(index for index in indexes if index.name == name) + except StopIteration: + raise ContextGroundingIndexNotFoundError(name) from None @traced(name="contextgrounding_list", run_type="uipath") def list( @@ -381,7 +461,7 @@ def list( "GET", Endpoint("/ecs_/v2/indexes"), params={"$expand": "dataSource"}, - headers={**header_folder(folder_key, None)}, + headers={**header_folder(folder_key, None), **header_job_key()}, ).json() return [ ContextGroundingIndex.model_validate(item) @@ -409,7 +489,7 @@ async def list_async( "GET", Endpoint("/ecs_/v2/indexes"), params={"$expand": "dataSource"}, - headers={**header_folder(folder_key, None)}, + headers={**header_folder(folder_key, None), **header_job_key()}, ) ).json() return [ @@ -447,6 +527,7 @@ def retrieve_by_id( spec.method, spec.endpoint, params=spec.params, + headers=spec.headers, ).json() @traced(name="contextgrounding_retrieve_by_id", run_type="uipath") @@ -479,6 +560,7 @@ async def retrieve_by_id_async( spec.method, spec.endpoint, params=spec.params, + headers=spec.headers, ) return response.json() @@ -598,20 +680,29 @@ async def create_index_async( @resource_override(resource_type="index") @traced(name="contextgrounding_create_ephemeral_index", run_type="uipath") def create_ephemeral_index( - self, usage: EphemeralIndexUsage, attachments: List[str] + self, + usage: EphemeralIndexUsage, + attachments: List[str], + folder_key: str | None = None, + folder_path: str | None = None, ) -> ContextGroundingIndex: """Create a new ephemeral context grounding index. Args: usage (EphemeralIndexUsage): The task type for the ephemeral index (DeepRAG or BatchRAG) attachments (list[str]): The list of attachments ids from which the ephemeral index will be created + folder_key (Optional[str]): The folder key to scope the ephemeral index to. + folder_path (Optional[str]): The folder path to scope the ephemeral index to (resolved to a key if folder_key is not provided). Returns: ContextGroundingIndex: The created index information. """ + if folder_key is not None or folder_path is not None: + folder_key = self._resolve_folder_key(folder_key, folder_path) spec = self._create_ephemeral_spec( usage, attachments, + folder_key=folder_key, ) response = self.request( @@ -626,20 +717,29 @@ def create_ephemeral_index( @resource_override(resource_type="index") @traced(name="contextgrounding_create_ephemeral_index", run_type="uipath") async def create_ephemeral_index_async( - self, usage: EphemeralIndexUsage, attachments: List[str] + self, + usage: EphemeralIndexUsage, + attachments: List[str], + folder_key: str | None = None, + folder_path: str | None = None, ) -> ContextGroundingIndex: """Create a new ephemeral context grounding index. Args: usage (EphemeralIndexUsage): The task type for the ephemeral index (DeepRAG or BatchRAG) attachments (list[str]): The list of attachments ids from which the ephemeral index will be created + folder_key (Optional[str]): The folder key to scope the ephemeral index to. + folder_path (Optional[str]): The folder path to scope the ephemeral index to (resolved to a key if folder_key is not provided). Returns: ContextGroundingIndex: The created index information. """ + if folder_key is not None or folder_path is not None: + folder_key = self._resolve_folder_key(folder_key, folder_path) spec = self._create_ephemeral_spec( usage, attachments, + folder_key=folder_key, ) response = await self.request_async( @@ -1032,6 +1132,10 @@ def download_batch_transform_result( batch_transform = self.retrieve_batch_transform( id=id, index_name=index_name ) + if batch_transform.last_batch_rag_status == BatchTransformStatus.FAILED: + raise BatchTransformFailedException( + batch_transform_id=id, + ) if batch_transform.last_batch_rag_status != BatchTransformStatus.SUCCESSFUL: raise BatchTransformNotCompleteException( batch_transform_id=id, @@ -1091,6 +1195,10 @@ async def download_batch_transform_result_async( batch_transform = await self.retrieve_batch_transform_async( id=id, index_name=index_name ) + if batch_transform.last_batch_rag_status == BatchTransformStatus.FAILED: + raise BatchTransformFailedException( + batch_transform_id=id, + ) if batch_transform.last_batch_rag_status != BatchTransformStatus.SUCCESSFUL: raise BatchTransformNotCompleteException( batch_transform_id=id, @@ -1456,12 +1564,13 @@ def unified_search( self, name: str, query: str, - search_mode: SearchMode = SearchMode.AUTO, + search_mode: SearchMode = SearchMode.SEMANTIC, number_of_results: int = 10, threshold: float = 0.0, scope: Optional[UnifiedSearchScope] = None, folder_key: Optional[str] = None, folder_path: Optional[str] = None, + include_system_indexes: bool = False, ) -> UnifiedQueryResult: """Perform a unified search on a context grounding index. @@ -1471,17 +1580,25 @@ def unified_search( Args: name (str): The name of the context index to search in. query (str): The search query in natural language. - search_mode (SearchMode): The search mode to use. Defaults to AUTO. + search_mode (SearchMode): The search mode to use. Defaults to SEMANTIC. number_of_results (int): Maximum number of results to return. Defaults to 10. threshold (float): Minimum similarity threshold. Defaults to 0.0. scope (Optional[UnifiedSearchScope]): Optional search scope (folder, extension). folder_key (Optional[str]): The key of the folder where the index resides. folder_path (Optional[str]): The path of the folder where the index resides. + include_system_indexes (bool): If True, fall back to tenant-wide + system indexes when the index is not found in folder or + across-folders listings. Defaults to False. Returns: UnifiedQueryResult: The unified search result containing semantic and/or tabular results. """ - index = self.retrieve(name, folder_key=folder_key, folder_path=folder_path) + index = self.retrieve( + name, + folder_key=folder_key, + folder_path=folder_path, + include_system_indexes=include_system_indexes, + ) folder_key = folder_key or index.folder_key @@ -1511,12 +1628,13 @@ async def unified_search_async( self, name: str, query: str, - search_mode: SearchMode = SearchMode.AUTO, + search_mode: SearchMode = SearchMode.SEMANTIC, number_of_results: int = 10, threshold: float = 0.0, scope: Optional[UnifiedSearchScope] = None, folder_key: Optional[str] = None, folder_path: Optional[str] = None, + include_system_indexes: bool = False, ) -> UnifiedQueryResult: """Asynchronously perform a unified search on a context grounding index. @@ -1526,18 +1644,24 @@ async def unified_search_async( Args: name (str): The name of the context index to search in. query (str): The search query in natural language. - search_mode (SearchMode): The search mode to use. Defaults to AUTO. + search_mode (SearchMode): The search mode to use. Defaults to SEMANTIC. number_of_results (int): Maximum number of results to return. Defaults to 10. threshold (float): Minimum similarity threshold. Defaults to 0.0. scope (Optional[UnifiedSearchScope]): Optional search scope (folder, extension). folder_key (Optional[str]): The key of the folder where the index resides. folder_path (Optional[str]): The path of the folder where the index resides. + include_system_indexes (bool): If True, fall back to tenant-wide + system indexes when the index is not found in folder or + across-folders listings. Defaults to False. Returns: UnifiedQueryResult: The unified search result containing semantic and/or tabular results. """ index = await self.retrieve_async( - name, folder_key=folder_key, folder_path=folder_path + name, + folder_key=folder_key, + folder_path=folder_path, + include_system_indexes=include_system_indexes, ) if index and index.in_progress_ingestion(): raise IngestionInProgressException(index_name=name) @@ -1881,9 +2005,20 @@ def _ingest_spec( endpoint=Endpoint(f"/ecs_/v2/indexes/{key}/ingest"), headers={ **header_folder(folder_key, None), + **header_job_key(), }, ) + @staticmethod + def _odata_name_filter(name: str) -> str: + """Build an OData ``Name eq ''`` filter with single quotes escaped. + + OData string literals escape ``'`` by doubling it. URL encoding of the + resulting filter is handled by the HTTP client when params are passed + as a dict. + """ + return "Name eq '{}'".format(name.replace("'", "''")) + def _retrieve_across_folders_spec( self, name: Optional[str] = None, @@ -1892,12 +2027,30 @@ def _retrieve_across_folders_spec( "$expand": "dataSource", } if name: - params["$filter"] = f"Name eq '{name}'" + params["$filter"] = self._odata_name_filter(name) return RequestSpec( method="GET", endpoint=Endpoint("/ecs_/v2/indexes/allacrossfolders"), params=params, + headers={**header_job_key()}, + ) + + def _retrieve_system_indexes_spec( + self, + name: Optional[str] = None, + ) -> RequestSpec: + params: Dict[str, str] = { + "$expand": "dataSource", + } + if name: + params["$filter"] = self._odata_name_filter(name) + + return RequestSpec( + method="GET", + endpoint=Endpoint("/ecs_/v2/indexes/allsystemindexes"), + params=params, + headers={**header_job_key()}, ) def _list_spec( @@ -1912,6 +2065,7 @@ def _list_spec( }, headers={ **header_folder(folder_key, None), + **header_job_key(), }, ) @@ -1927,11 +2081,12 @@ def _retrieve_spec( method="GET", endpoint=Endpoint("/ecs_/v2/indexes"), params={ - "$filter": f"Name eq '{name}'", + "$filter": self._odata_name_filter(name), "$expand": "dataSource", }, headers={ **header_folder(folder_key, None), + **header_job_key(), }, ) @@ -1984,6 +2139,7 @@ def _create_spec( json=payload.model_dump(by_alias=True, exclude_none=True), headers={ **header_folder(folder_key, None), + **header_job_key(), }, ) @@ -1991,12 +2147,14 @@ def _create_ephemeral_spec( self, usage: str, attachments: List[str], + folder_key: str | None = None, ) -> RequestSpec: """Create request spec for ephemeral index creation. Args: usage (str): The task in which the ephemeral index will be used for attachments (list[str]): The list of attachments ids from which the ephemeral index will be created + folder_key (Optional[str]): The folder key to scope the ephemeral index to. Returns: RequestSpec for the create index request @@ -2012,7 +2170,7 @@ def _create_ephemeral_spec( method="POST", endpoint=Endpoint("/ecs_/v2/indexes/createephemeral"), json=payload.model_dump(by_alias=True, exclude_none=True), - headers={}, + headers={**header_folder(folder_key, None), **header_job_key()}, ) def _build_data_source(self, source: SourceConfig) -> Dict[str, Any]: @@ -2112,6 +2270,7 @@ def _retrieve_by_id_spec( endpoint=Endpoint(f"/ecs_/v2/indexes/{id}"), headers={ **header_folder(folder_key, None), + **header_job_key(), }, ) @@ -2128,6 +2287,7 @@ def _delete_by_id_spec( endpoint=Endpoint(f"/ecs_/v2/indexes/{id}"), headers={ **header_folder(folder_key, None), + **header_job_key(), }, ) @@ -2155,6 +2315,7 @@ def _search_spec( }, headers={ **header_folder(folder_key, None), + **header_job_key(), }, ) @@ -2162,7 +2323,7 @@ def _unified_search_spec( self, index_id: str, query: str, - search_mode: SearchMode = SearchMode.AUTO, + search_mode: SearchMode = SearchMode.SEMANTIC, number_of_results: int = 10, threshold: float = 0.0, scope: Optional[UnifiedSearchScope] = None, @@ -2191,6 +2352,7 @@ def _unified_search_spec( json=json_body, headers={ **header_folder(folder_key, None), + **header_job_key(), }, ) @@ -2220,6 +2382,7 @@ def _deep_rag_creation_spec( }, headers={ **header_folder(folder_key, None), + **header_job_key(), }, ) @@ -2243,7 +2406,7 @@ def _deep_rag_ephemeral_creation_spec( params={ "$select": "id,lastDeepRagStatus,createdDate", }, - headers={}, + headers={**header_job_key()}, ) def _batch_transform_creation_spec( @@ -2293,6 +2456,7 @@ def _batch_transform_creation_spec( }, headers={ **header_folder(folder_key, None), + **header_job_key(), }, ) @@ -2319,7 +2483,7 @@ def _batch_transform_ephemeral_creation_spec( column.model_dump(by_alias=True) for column in output_columns ], }, - headers={}, + headers={**header_job_key()}, ) def _deep_rag_retrieve_spec( @@ -2332,6 +2496,7 @@ def _deep_rag_retrieve_spec( params={ "$expand": "content", }, + headers={**header_job_key()}, ) def _batch_transform_retrieve_spec( @@ -2341,6 +2506,7 @@ def _batch_transform_retrieve_spec( return RequestSpec( method="GET", endpoint=Endpoint(f"/ecs_/v2/batchRag/{id}"), + headers={**header_job_key()}, ) def _batch_transform_get_read_uri_spec( @@ -2350,6 +2516,7 @@ def _batch_transform_get_read_uri_spec( return RequestSpec( method="GET", endpoint=Endpoint(f"/ecs_/v2/batchRag/{id}/GetReadUri"), + headers={**header_job_key()}, ) def _batch_transform_download_blob_spec( @@ -2359,6 +2526,7 @@ def _batch_transform_download_blob_spec( return RequestSpec( method="GET", endpoint=Endpoint(f"/ecs_/v2/batchRag/{id}/DownloadBlob"), + headers={**header_job_key()}, ) def _resolve_folder_key(self, folder_key, folder_path): diff --git a/packages/uipath-platform/src/uipath/platform/context_grounding/context_grounding.py b/packages/uipath-platform/src/uipath/platform/context_grounding/context_grounding.py index fdcf5eac9..c5c0915bb 100644 --- a/packages/uipath-platform/src/uipath/platform/context_grounding/context_grounding.py +++ b/packages/uipath-platform/src/uipath/platform/context_grounding/context_grounding.py @@ -239,7 +239,6 @@ class ContextGroundingSearchResultItem(BaseModel): class SearchMode(str, Enum): """Enum representing possible unified search modes.""" - AUTO = "Auto" SEMANTIC = "Semantic" diff --git a/packages/uipath-platform/src/uipath/platform/context_grounding/context_grounding_payloads.py b/packages/uipath-platform/src/uipath/platform/context_grounding/context_grounding_payloads.py index a28903462..2dbaf63bd 100644 --- a/packages/uipath-platform/src/uipath/platform/context_grounding/context_grounding_payloads.py +++ b/packages/uipath-platform/src/uipath/platform/context_grounding/context_grounding_payloads.py @@ -6,7 +6,7 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator from pydantic.alias_generators import to_camel -from uipath.platform.common.constants import ( +from uipath.platform.constants import ( CONFLUENCE_DATA_SOURCE_REQUEST, DROPBOX_DATA_SOURCE_REQUEST, GOOGLE_DRIVE_DATA_SOURCE_REQUEST, diff --git a/packages/uipath-platform/src/uipath/platform/entities/__init__.py b/packages/uipath-platform/src/uipath/platform/entities/__init__.py index bbc43cdb7..8c323eca3 100644 --- a/packages/uipath-platform/src/uipath/platform/entities/__init__.py +++ b/packages/uipath-platform/src/uipath/platform/entities/__init__.py @@ -4,37 +4,81 @@ """ from ._entities_service import EntitiesService +from ._entity_ontology_service import DataFabricOntologyItem from .entities import ( + AggregateRow, + ChoiceSetValue, + DataFabricEntityItem, Entity, + EntityAggregate, + EntityAggregateFunction, + EntityBinning, + EntityCreateFieldOptions, + EntityCreateOptions, EntityField, + EntityFieldDataType, EntityFieldMetadata, + EntityImportRecordsResponse, + EntityJoin, + EntityMetadataUpdateOptions, + EntityQueryFilter, + EntityQueryFilterGroup, + EntityQuerySortOption, EntityRecord, EntityRecordsBatchResponse, + EntityRecordsListResponse, EntityRouting, + EntitySetResolution, ExternalField, ExternalObject, ExternalSourceFields, + FailureRecord, FieldDataType, FieldMetadata, + LogicalOperator, + QueryFilterOperator, QueryRoutingOverrideContext, ReferenceType, + RetrieveEntityRecordsResponse, SourceJoinCriteria, ) __all__ = [ + "AggregateRow", + "ChoiceSetValue", + "DataFabricEntityItem", + "DataFabricOntologyItem", "EntitiesService", "Entity", + "EntityAggregate", + "EntityAggregateFunction", + "EntityBinning", + "EntityCreateFieldOptions", + "EntityCreateOptions", "EntityField", - "EntityRecord", + "EntityFieldDataType", "EntityFieldMetadata", - "EntityRouting", - "FieldDataType", - "FieldMetadata", + "EntityImportRecordsResponse", + "EntityJoin", + "EntityMetadataUpdateOptions", + "EntityQueryFilter", + "EntityQueryFilterGroup", + "EntityQuerySortOption", + "EntityRecord", "EntityRecordsBatchResponse", + "EntityRecordsListResponse", + "EntityRouting", + "EntitySetResolution", "ExternalField", "ExternalObject", "ExternalSourceFields", + "FailureRecord", + "FieldDataType", + "FieldMetadata", + "LogicalOperator", + "QueryFilterOperator", "QueryRoutingOverrideContext", "ReferenceType", + "RetrieveEntityRecordsResponse", "SourceJoinCriteria", ] diff --git a/packages/uipath-platform/src/uipath/platform/entities/_entities_service.py b/packages/uipath-platform/src/uipath/platform/entities/_entities_service.py index f30c9492e..fa2845e63 100644 --- a/packages/uipath-platform/src/uipath/platform/entities/_entities_service.py +++ b/packages/uipath-platform/src/uipath/platform/entities/_entities_service.py @@ -1,55 +1,133 @@ +"""Public facade for the Data Fabric entities surface. + +:class:`EntitiesService` keeps the existing ``sdk.entities.*`` API flat and +unchanged from a caller's perspective while delegating each operation to the +appropriate underlying service: + +* :class:`EntitySchemaService` — entity definitions, choice set listings, + create / delete / update-metadata lifecycle. +* :class:`EntityDataService` — record CRUD (single and batch), structured + queries, attachments, choice-set values, bulk import, and federated SQL + queries. + +The facade additionally owns cross-cutting concerns such as agent entity-set +resolution. +""" + +import logging from typing import Any, Dict, List, Optional, Type -import sqlparse from httpx import Response -from sqlparse.sql import Parenthesis, Where -from sqlparse.tokens import DML, Keyword, Wildcard from uipath.core.tracing import traced from ..common._base_service import BaseService +from ..common._bindings import _resource_overwrites from ..common._config import UiPathApiConfig from ..common._execution_context import UiPathExecutionContext -from ..common._models import Endpoint, RequestSpec +from ..errors._datafabric_error import attach_datafabric_error_mapping +from ..orchestrator._folder_service import FolderService +from ._entity_data_service import EntityDataService, FileContent +from ._entity_ontology_service import EntityOntologyService +from ._entity_resolution import ( + build_resolution_service, + create_resolution_plan, + create_resolution_plan_async, + create_routing_strategy, + fetch_resolved_entities, + fetch_resolved_entities_async, +) +from ._entity_schema_service import EntitySchemaService from .entities import ( + ChoiceSetValue, + DataFabricEntityItem, Entity, + EntityAggregate, + EntityBinning, + EntityCreateFieldOptions, + EntityCreateOptions, + EntityImportRecordsResponse, + EntityJoin, + EntityMetadataUpdateOptions, + EntityQueryFilterGroup, + EntityQuerySortOption, EntityRecord, EntityRecordsBatchResponse, + EntityRecordsListResponse, + EntitySetResolution, QueryRoutingOverrideContext, + RetrieveEntityRecordsResponse, ) -_FORBIDDEN_DML = {"INSERT", "UPDATE", "DELETE", "MERGE", "REPLACE"} -_FORBIDDEN_DDL = {"DROP", "ALTER", "CREATE", "TRUNCATE"} -_DISALLOWED_KEYWORDS = [ - "WITH", - "UNION", - "INTERSECT", - "EXCEPT", - "OVER", - "ROLLUP", - "CUBE", - "GROUPING", - "PARTITION", -] +logger = logging.getLogger(__name__) class EntitiesService(BaseService): """Service for managing UiPath Data Service entities. - Entities are database tables in UiPath Data Service that can store - structured data for automation processes. + Entities are database tables in UiPath Data Service that store structured + data for automation processes. This service is the unified entry point for + every entity operation: schema management, record CRUD, structured and + SQL queries, file attachments, choice sets, and bulk import. See Also: https://docs.uipath.com/data-service/automation-cloud/latest/user-guide/introduction !!! warning "Preview Feature" - This function is currently experimental. - Behavior and parameters are subject to change in future versions. + This service is currently experimental. Behavior and parameters are + subject to change in future versions. """ def __init__( - self, config: UiPathApiConfig, execution_context: UiPathExecutionContext + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + folders_service: Optional[FolderService] = None, + folders_map: Optional[Dict[str, str]] = None, + entity_name_overrides: Optional[Dict[str, str]] = None, + routing_context: Optional[QueryRoutingOverrideContext] = None, ) -> None: + """Initialise the facade and its underlying schema and data services.""" super().__init__(config=config, execution_context=execution_context) + self._folders_service = folders_service + self._routing_strategy = create_routing_strategy( + folders_map=folders_map, + effective_entity_names=entity_name_overrides, + routing_context=routing_context, + folders_service=folders_service, + ) + self._schema = EntitySchemaService( + config=config, + execution_context=execution_context, + folders_service=folders_service, + ) + self._data = EntityDataService( + config=config, + execution_context=execution_context, + folders_service=folders_service, + routing_strategy=self._routing_strategy, + ) + self._ontology = EntityOntologyService( + config=config, + execution_context=execution_context, + folders_service=folders_service, + ) + + async def aclose(self) -> None: + """Close this facade and the services it creates.""" + try: + await self._schema.aclose() + finally: + try: + await self._data.aclose() + finally: + try: + await self._ontology.aclose() + finally: + await super().aclose() + + # ------------------------------------------------------------------ + # Schema operations — delegate to EntitySchemaService + # ------------------------------------------------------------------ @traced(name="entity_retrieve", run_type="uipath") def retrieve(self, entity_key: str) -> Entity: @@ -84,10 +162,7 @@ def retrieve(self, entity_key: str) -> Entity: print(f" Required: {field.is_required}") print(f" Primary Key: {field.is_primary_key}") """ - spec = self._retrieve_spec(entity_key) - response = self.request(spec.method, spec.endpoint) - - return Entity.model_validate(response.json()) + return self._schema.retrieve(entity_key) @traced(name="entity_retrieve", run_type="uipath") async def retrieve_async(self, entity_key: str) -> Entity: @@ -122,11 +197,41 @@ async def retrieve_async(self, entity_key: str) -> Entity: print(f" Required: {field.is_required}") print(f" Primary Key: {field.is_primary_key}") """ - spec = self._retrieve_spec(entity_key) + return await self._schema.retrieve_async(entity_key) + + @traced(name="entity_retrieve_by_name", run_type="uipath") + def retrieve_by_name( + self, entity_name: str, folder_key: Optional[str] = None + ) -> Entity: + """Retrieve an entity by its name. - response = await self.request_async(spec.method, spec.endpoint) + The server resolves the entity within the folder identified by + ``folder_key``. When omitted the default folder from the + execution context is used. - return Entity.model_validate(response.json()) + Args: + entity_name: The name of the entity. + folder_key: Optional folder key for disambiguation. + """ + return self._schema.retrieve_by_name(entity_name, folder_key=folder_key) + + @traced(name="entity_retrieve_by_name", run_type="uipath") + async def retrieve_by_name_async( + self, entity_name: str, folder_key: Optional[str] = None + ) -> Entity: + """Asynchronously retrieve an entity by its name. + + The server resolves the entity within the folder identified by + ``folder_key``. When omitted the default folder from the + execution context is used. + + Args: + entity_name: The name of the entity. + folder_key: Optional folder key for disambiguation. + """ + return await self._schema.retrieve_by_name_async( + entity_name, folder_key=folder_key + ) @traced(name="list_entities", run_type="uipath") def list_entities(self) -> List[Entity]: @@ -165,11 +270,7 @@ def list_entities(self) -> List[Entity]: print(f"Total records: {total_records}") print(f"Total storage: {total_storage:.2f} MB") """ - spec = self._list_entities_spec() - response = self.request(spec.method, spec.endpoint) - - entities_data = response.json() - return [Entity.model_validate(entity) for entity in entities_data] + return self._schema.list_entities() @traced(name="list_entities", run_type="uipath") async def list_entities_async(self) -> List[Entity]: @@ -208,20 +309,295 @@ async def list_entities_async(self) -> List[Entity]: print(f"Total records: {total_records}") print(f"Total storage: {total_storage:.2f} MB") """ - spec = self._list_entities_spec() - response = await self.request_async(spec.method, spec.endpoint) + return await self._schema.list_entities_async() + + @traced(name="list_choicesets", run_type="uipath") + def list_choicesets(self) -> List[Entity]: + """List all choice sets in Data Service. + + Returns: + List[Entity]: A list of all choice set entities. + + Examples: + List all choice sets:: + + choicesets = entities_service.list_choicesets() + for cs in choicesets: + print(f"{cs.display_name} ({cs.id})") + """ + return self._schema.list_choicesets() + + @traced(name="list_choicesets", run_type="uipath") + async def list_choicesets_async(self) -> List[Entity]: + """Asynchronously list all choice sets in Data Service. + + Returns: + List[Entity]: A list of all choice set entities. + """ + return await self._schema.list_choicesets_async() + + @traced(name="entity_create", run_type="uipath") + def create_entity( + self, + name: str, + fields: List[EntityCreateFieldOptions], + options: Optional[EntityCreateOptions] = None, + ) -> str: + """Create a new entity with the given schema and return its id. + + Args: + name (str): Entity name. Must start with a letter and contain + only letters, digits, and underscores (3-100 characters). + fields (List[EntityCreateFieldOptions]): Field definitions for + the new entity. Each entry declares the field's name, type, + and optional constraints such as ``length_limit``, + ``decimal_precision``, ``is_required``, ``is_unique``, etc. + options (Optional[EntityCreateOptions]): Optional entity-level + settings such as display name, description, folder + placement, and RBAC / analytics flags. + + Returns: + str: The id (UUID) of the newly created entity. + + Raises: + ValueError: If the entity name or any field name fails the + client-side validation (regex / length / reserved names) or + if a per-field constraint is not supported for that field + type or is out of range. + + Examples: + Create a simple entity:: + + from uipath.platform.entities import ( + EntityCreateFieldOptions, + EntityCreateOptions, + EntityFieldDataType, + ) - entities_data = response.json() - return [Entity.model_validate(entity) for entity in entities_data] + entity_id = entities_service.create_entity( + "ProductCatalog", + [ + EntityCreateFieldOptions( + field_name="product_name", + type=EntityFieldDataType.STRING, + is_required=True, + is_unique=True, + ), + EntityCreateFieldOptions( + field_name="price", + type=EntityFieldDataType.DECIMAL, + decimal_precision=2, + ), + ], + options=EntityCreateOptions( + display_name="Product Catalog", + description="Inventory of available products", + is_rbac_enabled=True, + ), + ) + """ + return self._schema.create_entity(name, fields, options) + + @traced(name="entity_create", run_type="uipath") + async def create_entity_async( + self, + name: str, + fields: List[EntityCreateFieldOptions], + options: Optional[EntityCreateOptions] = None, + ) -> str: + """Asynchronously create a new entity with the given schema. + + Args: + name (str): Entity name; same validation rules as :meth:`create_entity`. + fields (List[EntityCreateFieldOptions]): Field definitions. + options (Optional[EntityCreateOptions]): Optional entity-level settings. + + Returns: + str: The id (UUID) of the newly created entity. + + Raises: + ValueError: For client-side validation failures. + + Examples: + Create a simple entity:: + + from uipath.platform.entities import ( + EntityCreateFieldOptions, + EntityFieldDataType, + ) + + entity_id = await entities_service.create_entity_async( + "ProductCatalog", + [ + EntityCreateFieldOptions( + field_name="product_name", + type=EntityFieldDataType.STRING, + is_required=True, + ), + ], + ) + """ + return await self._schema.create_entity_async(name, fields, options) + + @traced(name="entity_delete", run_type="uipath") + def delete_entity(self, entity_id: str) -> None: + """Delete an entity and all of its records. + + Args: + entity_id (str): The unique identifier of the entity to delete. + + Examples: + Delete an entity by id:: + + entities_service.delete_entity("a1b2c3d4-...") + """ + self._schema.delete_entity(entity_id) + + @traced(name="entity_delete", run_type="uipath") + async def delete_entity_async(self, entity_id: str) -> None: + """Asynchronously delete an entity and all of its records. + + Args: + entity_id (str): The unique identifier of the entity to delete. + + Examples: + Delete an entity by id:: + + await entities_service.delete_entity_async("a1b2c3d4-...") + """ + await self._schema.delete_entity_async(entity_id) + + @traced(name="entity_update_metadata", run_type="uipath") + def update_entity_metadata( + self, + entity_id: str, + metadata: EntityMetadataUpdateOptions | Dict[str, Any], + ) -> None: + """Update an entity's display name, description, and/or RBAC flag. + + Args: + entity_id (str): The unique identifier of the entity. + metadata (EntityMetadataUpdateOptions | Dict[str, Any]): + An :class:`EntityMetadataUpdateOptions` instance or a dict + with any of ``display_name``, ``description``, + ``is_rbac_enabled``. Dict keys may be snake_case + (``display_name``) or camelCase (``displayName``); both + serialize correctly to the API. + + Examples: + Rename and update description:: + + from uipath.platform.entities import EntityMetadataUpdateOptions + + entities_service.update_entity_metadata( + "a1b2c3d4-...", + EntityMetadataUpdateOptions( + display_name="New Display Name", + description="Refreshed description", + ), + ) + + From a plain dict:: + + entities_service.update_entity_metadata( + "a1b2c3d4-...", + {"display_name": "X", "is_rbac_enabled": True}, + ) + """ + self._schema.update_entity_metadata(entity_id, metadata) + + @traced(name="entity_update_metadata", run_type="uipath") + async def update_entity_metadata_async( + self, + entity_id: str, + metadata: EntityMetadataUpdateOptions | Dict[str, Any], + ) -> None: + """Asynchronously update an entity's display name, description, and/or RBAC flag. + + Args: + entity_id (str): The unique identifier of the entity. + metadata (EntityMetadataUpdateOptions | Dict[str, Any]): + An :class:`EntityMetadataUpdateOptions` instance or a dict + with any of ``display_name``, ``description``, + ``is_rbac_enabled``. + + Examples: + Rename:: + + from uipath.platform.entities import EntityMetadataUpdateOptions + + await entities_service.update_entity_metadata_async( + "a1b2c3d4-...", + EntityMetadataUpdateOptions(display_name="Renamed Entity"), + ) + """ + await self._schema.update_entity_metadata_async(entity_id, metadata) + + # ------------------------------------------------------------------ + # Data operations — delegate to EntityDataService + # ------------------------------------------------------------------ + + @traced(name="get_choiceset_values", run_type="uipath") + def get_choiceset_values( + self, + choiceset_id: str, + start: Optional[int] = None, + limit: Optional[int] = None, + ) -> List[ChoiceSetValue]: + """Get the values of a choice set by its ID. + + Args: + choiceset_id: The unique identifier of the choice set. + start: Optional offset for pagination. + limit: Optional page size for pagination. + + Returns: + List[ChoiceSetValue]: The values in the choice set, each containing + id, name, display_name, and number_id. + + Examples: + Get all values in a choice set:: + + values = entities_service.get_choiceset_values("choiceset-id") + for v in values: + print(f"{v.number_id}: {v.display_name}") + """ + return self._data.get_choiceset_values(choiceset_id, start=start, limit=limit) + + @traced(name="get_choiceset_values", run_type="uipath") + async def get_choiceset_values_async( + self, + choiceset_id: str, + start: Optional[int] = None, + limit: Optional[int] = None, + ) -> List[ChoiceSetValue]: + """Asynchronously get the values of a choice set by its ID. + + Args: + choiceset_id: The unique identifier of the choice set. + start: Optional offset for pagination. + limit: Optional page size for pagination. + + Returns: + List[ChoiceSetValue]: The values in the choice set. + """ + return await self._data.get_choiceset_values_async( + choiceset_id, start=start, limit=limit + ) @traced(name="entity_list_records", run_type="uipath") def list_records( self, entity_key: str, - schema: Optional[Type[Any]] = None, # Optional schema + schema: Optional[Type[Any]] = None, start: Optional[int] = None, limit: Optional[int] = None, - ) -> List[EntityRecord]: + expansion_level: Optional[int] = None, + filter: Optional[str] = None, + orderby: Optional[str] = None, + select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, + ) -> EntityRecordsListResponse: """List records from an entity with optional pagination and schema validation. The schema parameter enables type-safe access to entity records by validating the @@ -258,11 +634,23 @@ class CustomerRecord: start (Optional[int]): Starting index for pagination (0-based). limit (Optional[int]): Maximum number of records to return. + expansion_level (Optional[int]): Depth of foreign-key expansion in the + response (``0`` means no expansion). Higher values inline related + records up to that many hops. + filter (Optional[str]): OData ``$filter`` expression + (e.g. ``"status eq 'active'"``). + orderby (Optional[str]): OData ``$orderby`` expression + (e.g. ``"created_at desc"``). + select (Optional[List[str]]): Column projection — field names to + include (rendered as ``$select``). + expand (Optional[List[str]]): Relationship names to expand inline + (rendered as ``$expand``). Returns: - List[EntityRecord]: A list of entity records. Each record contains an 'id' field - and all other fields from the entity. Fields can be accessed as attributes - or dictionary keys on the EntityRecord object. + EntityRecordsListResponse: A list-compatible response with + ``total_count``, ``has_next_page`` and ``next_cursor`` pagination + metadata. Iteration, indexing, and ``len()`` continue to work + like a plain list of :class:`EntityRecord`. Raises: ValueError: If schema validation fails for any record, including cases where @@ -280,6 +668,22 @@ class CustomerRecord: # Get first 50 records records = entities_service.list_records("Customers", start=0, limit=50) + print(f"Showing {len(records)} of {records.total_count} total") + if records.has_next_page: + next_page = entities_service.list_records( + "Customers", start=50, limit=50 + ) + + With OData filter, sorting, projection, and expansion:: + + records = entities_service.list_records( + "Customers", + filter="status eq 'active'", + orderby="created_at desc", + select=["name", "email", "status"], + expand=["company"], + expansion_level=1, + ) With schema validation:: @@ -299,28 +703,31 @@ class CustomerRecord: for record in records: print(f"{record.name}: {record.email}") """ - # Example method to generate the API request specification (mocked here) - spec = self._list_records_spec(entity_key, start, limit) - - # Make the HTTP request (assumes self.request exists) - response = self.request(spec.method, spec.endpoint, params=spec.params) - - # Parse the response JSON and extract the "value" field - records_data = response.json().get("value", []) - - # Validate and wrap records - return [ - EntityRecord.from_data(data=record, model=schema) for record in records_data - ] + return self._data.list_records( + entity_key, + schema=schema, + start=start, + limit=limit, + expansion_level=expansion_level, + filter=filter, + orderby=orderby, + select=select, + expand=expand, + ) @traced(name="entity_list_records", run_type="uipath") async def list_records_async( self, entity_key: str, - schema: Optional[Type[Any]] = None, # Optional schema + schema: Optional[Type[Any]] = None, start: Optional[int] = None, limit: Optional[int] = None, - ) -> List[EntityRecord]: + expansion_level: Optional[int] = None, + filter: Optional[str] = None, + orderby: Optional[str] = None, + select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, + ) -> EntityRecordsListResponse: """Asynchronously list records from an entity with optional pagination and schema validation. The schema parameter enables type-safe access to entity records by validating the @@ -357,11 +764,23 @@ class CustomerRecord: start (Optional[int]): Starting index for pagination (0-based). limit (Optional[int]): Maximum number of records to return. + expansion_level (Optional[int]): Depth of foreign-key expansion in the + response (``0`` means no expansion). Higher values inline related + records up to that many hops. + filter (Optional[str]): OData ``$filter`` expression + (e.g. ``"status eq 'active'"``). + orderby (Optional[str]): OData ``$orderby`` expression + (e.g. ``"created_at desc"``). + select (Optional[List[str]]): Column projection — field names to + include (rendered as ``$select``). + expand (Optional[List[str]]): Relationship names to expand inline + (rendered as ``$expand``). Returns: - List[EntityRecord]: A list of entity records. Each record contains an 'id' field - and all other fields from the entity. Fields can be accessed as attributes - or dictionary keys on the EntityRecord object. + EntityRecordsListResponse: A list-compatible response with + ``total_count``, ``has_next_page`` and ``next_cursor`` pagination + metadata. Iteration, indexing, and ``len()`` continue to work + like a plain list of :class:`EntityRecord`. Raises: ValueError: If schema validation fails for any record, including cases where @@ -379,6 +798,22 @@ class CustomerRecord: # Get first 50 records records = await entities_service.list_records_async("Customers", start=0, limit=50) + print(f"Showing {len(records)} of {records.total_count} total") + if records.has_next_page: + next_page = await entities_service.list_records_async( + "Customers", start=50, limit=50 + ) + + With OData filter, sorting, projection, and expansion:: + + records = await entities_service.list_records_async( + "Customers", + filter="status eq 'active'", + orderby="created_at desc", + select=["name", "email", "status"], + expand=["company"], + expansion_level=1, + ) With schema validation:: @@ -398,100 +833,315 @@ class CustomerRecord: for record in records: print(f"{record.name}: {record.email}") """ - spec = self._list_records_spec(entity_key, start, limit) + return await self._data.list_records_async( + entity_key, + schema=schema, + start=start, + limit=limit, + expansion_level=expansion_level, + filter=filter, + orderby=orderby, + select=select, + expand=expand, + ) + + @traced(name="entity_insert_record", run_type="uipath") + def insert_record( + self, + entity_key: str, + data: Any, + expansion_level: Optional[int] = None, + ) -> EntityRecord: + """Insert a single record into an entity and return the inserted row. + + Note: + Unlike :meth:`insert_records` (batch), this single-record endpoint + fires Data Fabric trigger events. Use this method when triggers + attached to the entity must run. + + Args: + entity_key (str): The unique key/identifier of the entity. + data (Any): Record payload — a dict, a Pydantic model, an + :class:`EntityRecord`, or any object exposing ``__dict__``. + expansion_level (Optional[int]): Depth of foreign-key expansion in + the response (``0`` means no expansion). + + Returns: + EntityRecord: The inserted record with its server-assigned ``Id`` + plus any expanded relationships. + + Examples: + Insert from a dict:: + + record = entities_service.insert_record( + "Customers", + {"name": "Alice", "email": "alice@example.com"}, + ) + print(record.id) + + Insert from a Pydantic model:: + + class CustomerInput(BaseModel): + name: str + email: str - # Make the HTTP request (assumes self.request exists) - response = await self.request_async( - spec.method, spec.endpoint, params=spec.params + record = entities_service.insert_record( + "Customers", + CustomerInput(name="Bob", email="bob@example.com"), + expansion_level=1, + ) + """ + return self._data.insert_record( + entity_key, data, expansion_level=expansion_level ) - # Parse the response JSON and extract the "value" field - records_data = response.json().get("value", []) + @traced(name="entity_insert_record", run_type="uipath") + async def insert_record_async( + self, + entity_key: str, + data: Any, + expansion_level: Optional[int] = None, + ) -> EntityRecord: + """Asynchronously insert a single record into an entity. - # Validate and wrap records - return [ - EntityRecord.from_data(data=record, model=schema) for record in records_data - ] + Note: + Unlike :meth:`insert_records_async` (batch), this single-record + endpoint fires Data Fabric trigger events. Use this method when + triggers attached to the entity must run. - @traced(name="entity_query_records", run_type="uipath") - def query_entity_records( + Args: + entity_key (str): The unique key/identifier of the entity. + data (Any): Record payload — a dict, a Pydantic model, an + :class:`EntityRecord`, or any object exposing ``__dict__``. + expansion_level (Optional[int]): Depth of foreign-key expansion in + the response (``0`` means no expansion). + + Returns: + EntityRecord: The inserted record with its server-assigned ``Id``. + + Examples: + Insert from a dict:: + + record = await entities_service.insert_record_async( + "Customers", + {"name": "Alice", "email": "alice@example.com"}, + ) + print(record.id) + """ + return await self._data.insert_record_async( + entity_key, data, expansion_level=expansion_level + ) + + @traced(name="entity_get_record", run_type="uipath") + def get_record( self, - sql_query: str, - routing_context: Optional[QueryRoutingOverrideContext] = None, - ) -> List[Dict[str, Any]]: - """Query entity records using a validated SQL query. + entity_key: str, + record_id: str, + expansion_level: Optional[int] = None, + ) -> EntityRecord: + """Fetch a single entity record by its id. - PREVIEW: This method is in preview and may change in future releases. + Args: + entity_key (str): The unique key/identifier of the entity. + record_id (str): The unique identifier of the record to fetch. + expansion_level (Optional[int]): Depth of foreign-key expansion in + the response (``0`` means no expansion). + + Returns: + EntityRecord: The record, with optional expanded relationships. + + Examples: + Basic usage:: + + record = entities_service.get_record("Customers", "rec-1") + print(record.id, record.name) + + With FK expansion:: + + # Inline the related Company record on the returned Customer + record = entities_service.get_record( + "Customers", "rec-1", expansion_level=1 + ) + """ + return self._data.get_record( + entity_key, record_id, expansion_level=expansion_level + ) + + @traced(name="entity_get_record", run_type="uipath") + async def get_record_async( + self, + entity_key: str, + record_id: str, + expansion_level: Optional[int] = None, + ) -> EntityRecord: + """Asynchronously fetch a single entity record by its id. Args: - sql_query (str): A SQL SELECT query to execute against Data Service entities. - Only SELECT statements are allowed. Queries without WHERE must include - a LIMIT clause. Subqueries and multi-statement queries are not permitted. - routing_context (Optional[QueryRoutingOverrideContext]): Per-entity routing context - for multi-folder queries. When present, included in the request body - and takes precedence over the folder header on the backend. + entity_key (str): The unique key/identifier of the entity. + record_id (str): The unique identifier of the record to fetch. + expansion_level (Optional[int]): Depth of foreign-key expansion in + the response (``0`` means no expansion). Returns: - List[Dict[str, Any]]: A list of result records as dictionaries. + EntityRecord: The record. - Raises: - ValueError: If the SQL query fails validation (e.g., non-SELECT, missing - WHERE/LIMIT, forbidden keywords, subqueries). + Examples: + Basic usage:: + + record = await entities_service.get_record_async("Customers", "rec-1") + print(record.id, record.name) """ - return self._query_entities_for_records( - sql_query, routing_context=routing_context + return await self._data.get_record_async( + entity_key, record_id, expansion_level=expansion_level ) - @traced(name="entity_query_records", run_type="uipath") - async def query_entity_records_async( + @traced(name="entity_update_record", run_type="uipath") + def update_record( self, - sql_query: str, - routing_context: Optional[QueryRoutingOverrideContext] = None, - ) -> List[Dict[str, Any]]: - """Asynchronously query entity records using a validated SQL query. + entity_key: str, + record_id: str, + data: Any, + expansion_level: Optional[int] = None, + ) -> EntityRecord: + """Update a single record by id and return the updated row. - PREVIEW: This method is in preview and may change in future releases. + Note: + Unlike :meth:`update_records` (batch), this single-record endpoint + fires Data Fabric trigger events. Use this method when triggers + attached to the entity must run. Args: - sql_query (str): A SQL SELECT query to execute against Data Service entities. - Only SELECT statements are allowed. Queries without WHERE must include - a LIMIT clause. Subqueries and multi-statement queries are not permitted. - routing_context (Optional[QueryRoutingOverrideContext]): Per-entity routing context - for multi-folder queries. When present, included in the request body - and takes precedence over the folder header on the backend. + entity_key (str): The unique key/identifier of the entity. + record_id (str): The unique identifier of the record to update. + data (Any): Fields to update — a dict, a Pydantic model, or any + object exposing ``__dict__``. Fields explicitly set to + ``None`` are sent through; unset fields are omitted. + expansion_level (Optional[int]): Depth of foreign-key expansion in + the response (``0`` means no expansion). Returns: - List[Dict[str, Any]]: A list of result records as dictionaries. + EntityRecord: The updated record. - Raises: - ValueError: If the SQL query fails validation (e.g., non-SELECT, missing - WHERE/LIMIT, forbidden keywords, subqueries). + Examples: + Partial update from a dict:: + + record = entities_service.update_record( + "Customers", + "rec-1", + {"email": "alice.new@example.com"}, + ) + + Clear a field by passing an explicit ``None``:: + + # Note: unset fields are omitted; explicit None values are sent. + record = entities_service.update_record( + "Customers", + "rec-1", + {"middle_name": None}, + ) """ - return await self._query_entities_for_records_async( - sql_query, routing_context=routing_context + return self._data.update_record( + entity_key, record_id, data, expansion_level=expansion_level ) - def _query_entities_for_records( + @traced(name="entity_update_record", run_type="uipath") + async def update_record_async( self, - sql_query: str, - *, - routing_context: Optional[QueryRoutingOverrideContext] = None, - ) -> List[Dict[str, Any]]: - self._validate_sql_query(sql_query) - spec = self._query_entity_records_spec(sql_query, routing_context) - response = self.request(spec.method, spec.endpoint, json=spec.json) - return response.json().get("results", []) + entity_key: str, + record_id: str, + data: Any, + expansion_level: Optional[int] = None, + ) -> EntityRecord: + """Asynchronously update a single record by id. + + Note: + Unlike :meth:`update_records_async` (batch), this single-record + endpoint fires Data Fabric trigger events. + + Args: + entity_key (str): The unique key/identifier of the entity. + record_id (str): The unique identifier of the record to update. + data (Any): Fields to update — a dict, a Pydantic model, or any + object exposing ``__dict__``. + expansion_level (Optional[int]): Depth of foreign-key expansion. + + Returns: + EntityRecord: The updated record. + + Examples: + Partial update:: + + record = await entities_service.update_record_async( + "Customers", + "rec-1", + {"email": "alice.new@example.com"}, + ) + """ + return await self._data.update_record_async( + entity_key, record_id, data, expansion_level=expansion_level + ) + + @traced(name="entity_delete_record", run_type="uipath") + def delete_record(self, entity_key: str, record_id: str) -> None: + """Delete a single record by id. + + Note: + Unlike :meth:`delete_records` (batch), this single-record endpoint + fires Data Fabric trigger events. Use this method when triggers + attached to the entity must run on delete. + + Args: + entity_key (str): The unique key/identifier of the entity. + record_id (str): The unique identifier of the record to delete. + + Examples: + Delete by id:: + + entities_service.delete_record("Customers", "rec-1") + """ + self._data.delete_record(entity_key, record_id) + + @traced(name="entity_delete_record", run_type="uipath") + async def delete_record_async(self, entity_key: str, record_id: str) -> None: + """Asynchronously delete a single record by id. - async def _query_entities_for_records_async( + Note: + Unlike :meth:`delete_records_async` (batch), this single-record + endpoint fires Data Fabric trigger events. + + Args: + entity_key (str): The unique key/identifier of the entity. + record_id (str): The unique identifier of the record to delete. + + Examples: + Delete by id:: + + await entities_service.delete_record_async("Customers", "rec-1") + """ + await self._data.delete_record_async(entity_key, record_id) + + async def get_ontology_file_async( self, - sql_query: str, - *, - routing_context: Optional[QueryRoutingOverrideContext] = None, - ) -> List[Dict[str, Any]]: - self._validate_sql_query(sql_query) - spec = self._query_entity_records_spec(sql_query, routing_context) - response = await self.request_async(spec.method, spec.endpoint, json=spec.json) - return response.json().get("results", []) + ontology_name: str, + file_type: str = "owl", + folder_key: Optional[str] = None, + ) -> Dict[str, Any]: + """Fetch one file of an ontology from Data Fabric. + + !!! warning "Preview Feature" + This method is currently experimental. Behavior and parameters are + subject to change in future versions. + + Args: + ontology_name (str): Name of the ontology. + file_type (str): The ontology file to fetch — one of owl, r2rml, + shacl, summary, context. + folder_key (Optional[str]): Key of the folder the ontology lives in. + + Returns: + Dict[str, Any]: The file record (e.g. ``content``, ``mediaType``). + """ + return await self._ontology.get_file_async(ontology_name, file_type, folder_key) @traced(name="entity_record_insert_batch", run_type="uipath") def insert_records( @@ -499,20 +1149,29 @@ def insert_records( entity_key: str, records: List[Any], schema: Optional[Type[Any]] = None, + expansion_level: Optional[int] = None, + fail_on_first: Optional[bool] = None, ) -> EntityRecordsBatchResponse: """Insert multiple records into an entity in a single batch operation. Args: entity_key (str): The unique key/identifier of the entity. - records (List[Any]): List of records to insert. Each record should be an object - with attributes matching the entity's field names. + records (List[Any]): List of records to insert. Each record may be + a dict, a Pydantic model, an :class:`EntityRecord`, or any + object exposing ``__dict__``. schema (Optional[Type[Any]]): Optional schema class for validation. When provided, validates that each record in the response matches the schema structure. + expansion_level (Optional[int]): Depth of foreign-key expansion in + the response (``0`` means no expansion). + fail_on_first (Optional[bool]): When ``True``, stop the batch on + the first per-record failure. When ``False`` (default), all + records are attempted and the response lists both + ``success_records`` and ``failure_records``. Returns: EntityRecordsBatchResponse: Response containing successful and failed record operations. - - success_records: List of successfully inserted EntityRecord objects - - failure_records: List of EntityRecord objects that failed to insert + - success_records: List of successfully inserted :class:`EntityRecord` objects + - failure_records: List of :class:`FailureRecord` describing per-record errors Examples: Insert records without schema:: @@ -536,6 +1195,15 @@ def __init__(self, name, email, age): print(f"Inserted: {len(response.success_records)}") print(f"Failed: {len(response.failure_records)}") + Insert with FK expansion and fail-fast:: + + response = entities_service.insert_records( + "Orders", + [{"product_id": "p-1", "qty": 3}, {"product_id": "p-2", "qty": 1}], + expansion_level=1, # inline the related Product on each response record + fail_on_first=True, # abort the batch at the first error + ) + Insert with schema validation:: class CustomerSchema: @@ -561,10 +1229,13 @@ def __init__(self, name, email, age): for record in response.success_records: print(f"Inserted: {record.name} (ID: {record.id})") """ - spec = self._insert_batch_spec(entity_key, records) - response = self.request(spec.method, spec.endpoint, json=spec.json) - - return self.validate_entity_batch(response, schema) + return self._data.insert_records( + entity_key, + records, + schema=schema, + expansion_level=expansion_level, + fail_on_first=fail_on_first, + ) @traced(name="entity_record_insert_batch", run_type="uipath") async def insert_records_async( @@ -572,20 +1243,29 @@ async def insert_records_async( entity_key: str, records: List[Any], schema: Optional[Type[Any]] = None, + expansion_level: Optional[int] = None, + fail_on_first: Optional[bool] = None, ) -> EntityRecordsBatchResponse: """Asynchronously insert multiple records into an entity in a single batch operation. Args: entity_key (str): The unique key/identifier of the entity. - records (List[Any]): List of records to insert. Each record should be an object - with attributes matching the entity's field names. + records (List[Any]): List of records to insert. Each record may be + a dict, a Pydantic model, an :class:`EntityRecord`, or any + object exposing ``__dict__``. schema (Optional[Type[Any]]): Optional schema class for validation. When provided, validates that each record in the response matches the schema structure. + expansion_level (Optional[int]): Depth of foreign-key expansion in + the response (``0`` means no expansion). + fail_on_first (Optional[bool]): When ``True``, stop the batch on + the first per-record failure. When ``False`` (default), all + records are attempted and the response lists both + ``success_records`` and ``failure_records``. Returns: EntityRecordsBatchResponse: Response containing successful and failed record operations. - - success_records: List of successfully inserted EntityRecord objects - - failure_records: List of EntityRecord objects that failed to insert + - success_records: List of successfully inserted :class:`EntityRecord` objects + - failure_records: List of :class:`FailureRecord` describing per-record errors Examples: Insert records without schema:: @@ -634,10 +1314,13 @@ def __init__(self, name, email, age): for record in response.success_records: print(f"Inserted: {record.name} (ID: {record.id})") """ - spec = self._insert_batch_spec(entity_key, records) - response = await self.request_async(spec.method, spec.endpoint, json=spec.json) - - return self.validate_entity_batch(response, schema) + return await self._data.insert_records_async( + entity_key, + records, + schema=schema, + expansion_level=expansion_level, + fail_on_first=fail_on_first, + ) @traced(name="entity_record_update_batch", run_type="uipath") def update_records( @@ -645,20 +1328,30 @@ def update_records( entity_key: str, records: List[Any], schema: Optional[Type[Any]] = None, + expansion_level: Optional[int] = None, + fail_on_first: Optional[bool] = None, ) -> EntityRecordsBatchResponse: """Update multiple records in an entity in a single batch operation. Args: entity_key (str): The unique key/identifier of the entity. - records (List[Any]): List of records to update. Each record must have an 'Id' field - and should be a Pydantic model with `model_dump()` method or similar object. + records (List[Any]): List of records to update. Each record must + include its ``Id`` field. A record may be a dict, a Pydantic + model, an :class:`EntityRecord`, or any object exposing + ``__dict__``. schema (Optional[Type[Any]]): Optional schema class for validation. When provided, validates that each record in the request and response matches the schema structure. + expansion_level (Optional[int]): Depth of foreign-key expansion in + the response (``0`` means no expansion). + fail_on_first (Optional[bool]): When ``True``, stop the batch on + the first per-record failure. When ``False`` (default), all + records are attempted and the response lists both + ``success_records`` and ``failure_records``. Returns: EntityRecordsBatchResponse: Response containing successful and failed record operations. - - success_records: List of successfully updated EntityRecord objects - - failure_records: List of EntityRecord objects that failed to update + - success_records: List of successfully updated :class:`EntityRecord` objects + - failure_records: List of :class:`FailureRecord` describing per-record errors Examples: Update records:: @@ -707,15 +1400,13 @@ class CustomerSchema: for record in response.success_records: print(f"Updated: {record.name}") """ - valid_records = [ - EntityRecord.from_data(data=record.model_dump(by_alias=True), model=schema) - for record in records - ] - - spec = self._update_batch_spec(entity_key, valid_records) - response = self.request(spec.method, spec.endpoint, json=spec.json) - - return self.validate_entity_batch(response, schema) + return self._data.update_records( + entity_key, + records, + schema=schema, + expansion_level=expansion_level, + fail_on_first=fail_on_first, + ) @traced(name="entity_record_update_batch", run_type="uipath") async def update_records_async( @@ -723,20 +1414,30 @@ async def update_records_async( entity_key: str, records: List[Any], schema: Optional[Type[Any]] = None, + expansion_level: Optional[int] = None, + fail_on_first: Optional[bool] = None, ) -> EntityRecordsBatchResponse: """Asynchronously update multiple records in an entity in a single batch operation. Args: entity_key (str): The unique key/identifier of the entity. - records (List[Any]): List of records to update. Each record must have an 'Id' field - and should be a Pydantic model with `model_dump()` method or similar object. + records (List[Any]): List of records to update. Each record must + include its ``Id`` field. A record may be a dict, a Pydantic + model, an :class:`EntityRecord`, or any object exposing + ``__dict__``. schema (Optional[Type[Any]]): Optional schema class for validation. When provided, validates that each record in the request and response matches the schema structure. + expansion_level (Optional[int]): Depth of foreign-key expansion in + the response (``0`` means no expansion). + fail_on_first (Optional[bool]): When ``True``, stop the batch on + the first per-record failure. When ``False`` (default), all + records are attempted and the response lists both + ``success_records`` and ``failure_records``. Returns: EntityRecordsBatchResponse: Response containing successful and failed record operations. - - success_records: List of successfully updated EntityRecord objects - - failure_records: List of EntityRecord objects that failed to update + - success_records: List of successfully updated :class:`EntityRecord` objects + - failure_records: List of :class:`FailureRecord` describing per-record errors Examples: Update records:: @@ -785,32 +1486,35 @@ class CustomerSchema: for record in response.success_records: print(f"Updated: {record.name}") """ - valid_records = [ - EntityRecord.from_data(data=record.model_dump(by_alias=True), model=schema) - for record in records - ] - - spec = self._update_batch_spec(entity_key, valid_records) - response = await self.request_async(spec.method, spec.endpoint, json=spec.json) - - return self.validate_entity_batch(response, schema) + return await self._data.update_records_async( + entity_key, + records, + schema=schema, + expansion_level=expansion_level, + fail_on_first=fail_on_first, + ) @traced(name="entity_record_delete_batch", run_type="uipath") def delete_records( self, entity_key: str, record_ids: List[str], + fail_on_first: Optional[bool] = None, ) -> EntityRecordsBatchResponse: """Delete multiple records from an entity in a single batch operation. Args: entity_key (str): The unique key/identifier of the entity. record_ids (List[str]): List of record IDs (GUIDs) to delete. + fail_on_first (Optional[bool]): When ``True``, stop the batch on + the first per-record failure. When ``False`` (default), all + records are attempted and the response lists both + ``success_records`` and ``failure_records``. Returns: EntityRecordsBatchResponse: Response containing successful and failed record operations. - - success_records: List of successfully deleted EntityRecord objects - - failure_records: List of EntityRecord objects that failed to delete + - success_records: List of successfully deleted :class:`EntityRecord` objects + - failure_records: List of :class:`FailureRecord` describing per-record errors Examples: Delete specific records by ID:: @@ -847,31 +1551,31 @@ def delete_records( ) print(f"Deleted {len(response.success_records)} inactive records") """ - spec = self._delete_batch_spec(entity_key, record_ids) - response = self.request(spec.method, spec.endpoint, json=spec.json) - - delete_records_response = EntityRecordsBatchResponse.model_validate( - response.json() + return self._data.delete_records( + entity_key, record_ids, fail_on_first=fail_on_first ) - return delete_records_response - @traced(name="entity_record_delete_batch", run_type="uipath") async def delete_records_async( self, entity_key: str, record_ids: List[str], + fail_on_first: Optional[bool] = None, ) -> EntityRecordsBatchResponse: """Asynchronously delete multiple records from an entity in a single batch operation. Args: entity_key (str): The unique key/identifier of the entity. record_ids (List[str]): List of record IDs (GUIDs) to delete. + fail_on_first (Optional[bool]): When ``True``, stop the batch on + the first per-record failure. When ``False`` (default), all + records are attempted and the response lists both + ``success_records`` and ``failure_records``. Returns: EntityRecordsBatchResponse: Response containing successful and failed record operations. - - success_records: List of successfully deleted EntityRecord objects - - failure_records: List of EntityRecord objects that failed to delete + - success_records: List of successfully deleted :class:`EntityRecord` objects + - failure_records: List of :class:`FailureRecord` describing per-record errors Examples: Delete specific records by ID:: @@ -908,212 +1612,679 @@ async def delete_records_async( ) print(f"Deleted {len(response.success_records)} inactive records") """ - spec = self._delete_batch_spec(entity_key, record_ids) - response = await self.request_async(spec.method, spec.endpoint, json=spec.json) - - delete_records_response = EntityRecordsBatchResponse.model_validate( - response.json() + return await self._data.delete_records_async( + entity_key, record_ids, fail_on_first=fail_on_first ) - return delete_records_response - - def validate_entity_batch( + @traced(name="entity_retrieve_records", run_type="uipath") + def retrieve_records( self, - batch_response: Response, - schema: Optional[Type[Any]] = None, - ) -> EntityRecordsBatchResponse: - # Validate the response format - insert_records_response = EntityRecordsBatchResponse.model_validate( - batch_response.json() - ) + entity_key: str, + filter_group: Optional[EntityQueryFilterGroup] = None, + sort_options: Optional[List[EntityQuerySortOption]] = None, + selected_fields: Optional[List[str]] = None, + expansions: Optional[List[Any]] = None, + expansion_level: Optional[int] = None, + aggregates: Optional[List[EntityAggregate]] = None, + group_by: Optional[List[str]] = None, + joins: Optional[List[EntityJoin]] = None, + binnings: Optional[List[EntityBinning]] = None, + start: Optional[int] = None, + limit: Optional[int] = None, + ) -> RetrieveEntityRecordsResponse: + """Retrieve records with structured filters, sorting, expansion, joins, and aggregates. - # Validate individual records - validated_successful_records = [ - EntityRecord.from_data( - data=successful_record.model_dump(by_alias=True), model=schema - ) - for successful_record in insert_records_response.success_records - ] + Routes to the V2 endpoint when ``binnings`` is provided (numeric/date + binning is gated by the ``enable-binning-on-query`` feature flag on + the backend). - validated_failed_records = [ - EntityRecord.from_data( - data=failed_record.model_dump(by_alias=True), model=schema - ) - for failed_record in insert_records_response.failure_records - ] + Args: + entity_key (str): The unique key/identifier of the entity. + filter_group (Optional[EntityQueryFilterGroup]): Nested filter + conditions combined with AND/OR. + sort_options (Optional[List[EntityQuerySortOption]]): Sort fields + and direction. + selected_fields (Optional[List[str]]): Column projection — field + names to include; omit to return all fields. + expansions (Optional[List[Any]]): Foreign-key relationships to + expand inline on each result record. + expansion_level (Optional[int]): Depth of expansion (sent as a + URL query param). + aggregates (Optional[List[EntityAggregate]]): Aggregate + expressions (``COUNT`` / ``SUM`` / ``AVG`` / ``MIN`` / + ``MAX``). Maximum 5 per query. + group_by (Optional[List[str]]): Fields to group aggregate results + by. Maximum 5; required when both ``aggregates`` and + ``selected_fields`` are supplied. + joins (Optional[List[EntityJoin]]): Cross-entity joins. Maximum + 3, all of the same type. + binnings (Optional[List[EntityBinning]]): Bucket numeric or date + group-by fields. Each entry's field must also appear in + ``group_by``. + start (Optional[int]): Records to skip (pagination offset). + limit (Optional[int]): Maximum number of records to return. + + Returns: + RetrieveEntityRecordsResponse: A response with ``items``, + ``total_count``, ``has_next_page``, and ``next_cursor``. + ``items`` is a list of :class:`EntityRecord` for plain + queries, or :class:`AggregateRow` when ``aggregates``, + ``group_by``, or ``binnings`` are used. ``next_cursor`` is + populated only when the backend returns one; otherwise + paginate by passing the next ``start``. + + Examples: + Filter + sort + projection:: + + from uipath.platform.entities import ( + EntityQueryFilter, + EntityQueryFilterGroup, + EntityQuerySortOption, + LogicalOperator, + QueryFilterOperator, + ) + + result = entities_service.retrieve_records( + "Customers", + filter_group=EntityQueryFilterGroup( + logical_operator=LogicalOperator.And, + query_filters=[ + EntityQueryFilter( + field_name="status", + operator=QueryFilterOperator.Equals, + value="active", + ) + ], + ), + sort_options=[ + EntityQuerySortOption(field_name="created_at", is_descending=True) + ], + selected_fields=["Id", "name", "email"], + start=0, + limit=50, + ) + print(f"Found {result.total_count} customers") + + Aggregates and group-by (counts per status):: + + from uipath.platform.entities import ( + EntityAggregate, + EntityAggregateFunction, + ) - return EntityRecordsBatchResponse( - success_records=validated_successful_records, - failure_records=validated_failed_records, + result = entities_service.retrieve_records( + "Customers", + selected_fields=["status"], + group_by=["status"], + aggregates=[ + EntityAggregate( + function=EntityAggregateFunction.Count, + field="Id", + alias="total", + ) + ], + ) + for row in result.items: + print(row.status, row.total) + """ + return self._data.retrieve_records( + entity_key, + filter_group=filter_group, + sort_options=sort_options, + selected_fields=selected_fields, + expansions=expansions, + expansion_level=expansion_level, + aggregates=aggregates, + group_by=group_by, + joins=joins, + binnings=binnings, + start=start, + limit=limit, ) - def _retrieve_spec( + @traced(name="entity_retrieve_records", run_type="uipath") + async def retrieve_records_async( self, entity_key: str, - ) -> RequestSpec: - return RequestSpec( - method="GET", - endpoint=Endpoint(f"datafabric_/api/Entity/{entity_key}"), + filter_group: Optional[EntityQueryFilterGroup] = None, + sort_options: Optional[List[EntityQuerySortOption]] = None, + selected_fields: Optional[List[str]] = None, + expansions: Optional[List[Any]] = None, + expansion_level: Optional[int] = None, + aggregates: Optional[List[EntityAggregate]] = None, + group_by: Optional[List[str]] = None, + joins: Optional[List[EntityJoin]] = None, + binnings: Optional[List[EntityBinning]] = None, + start: Optional[int] = None, + limit: Optional[int] = None, + ) -> RetrieveEntityRecordsResponse: + """Asynchronously retrieve records with structured filters, sorting, expansion, joins, and aggregates. + + Routes to the V2 endpoint when ``binnings`` is provided (numeric/date + binning is gated by the ``enable-binning-on-query`` feature flag on + the backend). + + Args: + entity_key (str): The unique key/identifier of the entity. + filter_group (Optional[EntityQueryFilterGroup]): Nested filter + conditions combined with AND/OR. + sort_options (Optional[List[EntityQuerySortOption]]): Sort fields + and direction. + selected_fields (Optional[List[str]]): Column projection — field + names to include; omit to return all fields. + expansions (Optional[List[Any]]): Foreign-key relationships to + expand inline on each result record. + expansion_level (Optional[int]): Depth of expansion. + aggregates (Optional[List[EntityAggregate]]): Aggregate + expressions. Maximum 5 per query. + group_by (Optional[List[str]]): Fields to group aggregate results + by. Maximum 5; required when both ``aggregates`` and + ``selected_fields`` are supplied. + joins (Optional[List[EntityJoin]]): Cross-entity joins. Maximum + 3, all of the same type. + binnings (Optional[List[EntityBinning]]): Bucket numeric or date + group-by fields. + start (Optional[int]): Records to skip (pagination offset). + limit (Optional[int]): Maximum number of records to return. + + Returns: + RetrieveEntityRecordsResponse: A response with ``items``, + ``total_count``, ``has_next_page``, and ``next_cursor``. + + Examples: + Filter + sort + pagination:: + + from uipath.platform.entities import ( + EntityQueryFilter, + EntityQueryFilterGroup, + QueryFilterOperator, + ) + + result = await entities_service.retrieve_records_async( + "Customers", + filter_group=EntityQueryFilterGroup( + query_filters=[ + EntityQueryFilter( + field_name="status", + operator=QueryFilterOperator.Equals, + value="active", + ) + ], + ), + start=0, + limit=25, + ) + print(f"{len(result.items)} of {result.total_count} customers") + """ + return await self._data.retrieve_records_async( + entity_key, + filter_group=filter_group, + sort_options=sort_options, + selected_fields=selected_fields, + expansions=expansions, + expansion_level=expansion_level, + aggregates=aggregates, + group_by=group_by, + joins=joins, + binnings=binnings, + start=start, + limit=limit, ) - def _list_entities_spec(self) -> RequestSpec: - return RequestSpec( - method="GET", - endpoint=Endpoint("datafabric_/api/Entity"), + @attach_datafabric_error_mapping("query_entity_records") + @traced(name="entity_query_records", run_type="uipath") + def query_entity_records( + self, sql_query: str, *, relationships_as_scalar: bool = False + ) -> List[Dict[str, Any]]: + """Query entity records using a validated SQL query. + + PREVIEW: This method is in preview and may change in future releases. + + Args: + sql_query (str): A SQL SELECT query to execute against Data Service entities. + Only SELECT statements are allowed. Queries without WHERE must include + a LIMIT clause. Subqueries and multi-statement queries are not permitted. + relationships_as_scalar (bool, optional): When ``True``, relationship (FK) + fields are typed as their underlying scalar id instead of ``VARIANT``, + so a query can join on ``relationshipField = Other.Id``. Sent as + ``queryOptions.relationshipsAsScalar`` in the request body. Defaults to + ``False`` (unchanged behaviour). + + Notes: + A routing context is always derived from the configured ``folders_map`` + when present and included in the request body. + + Returns: + List[Dict[str, Any]]: A list of result records as dictionaries. + + Raises: + ValueError: If the SQL query fails validation (e.g., non-SELECT, missing + WHERE/LIMIT, forbidden keywords, subqueries). + """ + return self._data.query_entity_records(sql_query, relationships_as_scalar) + + @attach_datafabric_error_mapping("query_entity_records_async") + @traced(name="entity_query_records", run_type="uipath") + async def query_entity_records_async( + self, sql_query: str, *, relationships_as_scalar: bool = False + ) -> List[Dict[str, Any]]: + """Asynchronously query entity records using a validated SQL query. + + PREVIEW: This method is in preview and may change in future releases. + + Args: + sql_query (str): A SQL SELECT query to execute against Data Service entities. + Only SELECT statements are allowed. Queries without WHERE must include + a LIMIT clause. Subqueries and multi-statement queries are not permitted. + relationships_as_scalar (bool, optional): When ``True``, relationship (FK) + fields are typed as their underlying scalar id instead of ``VARIANT``, + so a query can join on ``relationshipField = Other.Id``. Sent as + ``queryOptions.relationshipsAsScalar`` in the request body. Defaults to + ``False`` (unchanged behaviour). + + Notes: + A routing context is always derived from the configured ``folders_map`` + when present and included in the request body. + + Returns: + List[Dict[str, Any]]: A list of result records as dictionaries. + + Raises: + ValueError: If the SQL query fails validation (e.g., non-SELECT, missing + WHERE/LIMIT, forbidden keywords, subqueries). + """ + return await self._data.query_entity_records_async( + sql_query, relationships_as_scalar ) - def _list_records_spec( + @traced(name="entity_upload_attachment", run_type="uipath") + def upload_attachment( self, - entity_key: str, - start: Optional[int] = None, - limit: Optional[int] = None, - ) -> RequestSpec: - return RequestSpec( - method="GET", - endpoint=Endpoint( - f"datafabric_/api/EntityService/entity/{entity_key}/read" - ), - params=({"start": start, "limit": limit}), + entity_id: str, + record_id: str, + field_name: str, + file: Optional[FileContent] = None, + file_path: Optional[str] = None, + expansion_level: Optional[int] = None, + ) -> Dict[str, Any]: + """Upload a file attachment to a File-type field on a record. + + Provide exactly one of ``file`` (raw bytes) or ``file_path`` (path on + disk). + + Args: + entity_id (str): The unique identifier of the entity. + record_id (str): The unique identifier of the record whose + attachment field is being set. + field_name (str): Name of the File-type field on the entity. + file (Optional[FileContent]): Raw bytes (``bytes`` / + ``bytearray`` / ``memoryview``) of the file to upload. + Mutually exclusive with ``file_path``. + file_path (Optional[str]): Path to a local file to upload. + Mutually exclusive with ``file``. + expansion_level (Optional[int]): Optional FK expansion depth in + the response (``0`` means no expansion). + + Returns: + Dict[str, Any]: The decoded JSON response (typically the updated + record), or an empty dict when the response has no body. + + Examples: + Upload from raw bytes:: + + with open("contract.pdf", "rb") as f: + data = f.read() + entities_service.upload_attachment( + "Customers", "rec-1", "Contract", file=data + ) + + Upload from a path on disk:: + + entities_service.upload_attachment( + "Customers", "rec-1", "Contract", file_path="./contract.pdf" + ) + """ + return self._data.upload_attachment( + entity_id, + record_id, + field_name, + file=file, + file_path=file_path, + expansion_level=expansion_level, ) - def _query_entity_records_spec( + @traced(name="entity_upload_attachment", run_type="uipath") + async def upload_attachment_async( self, - sql_query: str, - routing_context: Optional[QueryRoutingOverrideContext] = None, - ) -> RequestSpec: - body: Dict[str, Any] = {"query": sql_query} - if routing_context: - body["routingContext"] = routing_context.model_dump( - by_alias=True, exclude_none=True - ) - return RequestSpec( - method="POST", - endpoint=Endpoint("datafabric_/api/v1/query/execute"), - json=body, + entity_id: str, + record_id: str, + field_name: str, + file: Optional[FileContent] = None, + file_path: Optional[str] = None, + expansion_level: Optional[int] = None, + ) -> Dict[str, Any]: + """Asynchronously upload a file attachment to a File-type field on a record. + + Provide exactly one of ``file`` (raw bytes) or ``file_path`` (path on + disk). + + Args: + entity_id (str): The unique identifier of the entity. + record_id (str): The unique identifier of the record whose + attachment field is being set. + field_name (str): Name of the File-type field on the entity. + file (Optional[FileContent]): Raw bytes of the file to upload. + Mutually exclusive with ``file_path``. + file_path (Optional[str]): Path to a local file to upload. + Mutually exclusive with ``file``. + expansion_level (Optional[int]): Optional FK expansion depth in + the response. + + Returns: + Dict[str, Any]: The decoded JSON response. + + Examples: + Upload from a path on disk:: + + await entities_service.upload_attachment_async( + "Customers", "rec-1", "Contract", file_path="./contract.pdf" + ) + """ + return await self._data.upload_attachment_async( + entity_id, + record_id, + field_name, + file=file, + file_path=file_path, + expansion_level=expansion_level, ) - def _insert_batch_spec(self, entity_key: str, records: List[Any]) -> RequestSpec: - return RequestSpec( - method="POST", - endpoint=Endpoint( - f"datafabric_/api/EntityService/entity/{entity_key}/insert-batch" - ), - json=[record.__dict__ for record in records], + @traced(name="entity_download_attachment", run_type="uipath") + def download_attachment( + self, entity_id: str, record_id: str, field_name: str + ) -> bytes: + """Download a file attached to a record and return its raw bytes. + + Args: + entity_id (str): The unique identifier of the entity. + record_id (str): The unique identifier of the record containing + the attachment. + field_name (str): Name of the File-type field on the entity. + + Returns: + bytes: The raw file content. + + Examples: + Save the downloaded bytes to disk:: + + content = entities_service.download_attachment( + "Customers", "rec-1", "Contract" + ) + with open("downloaded.pdf", "wb") as f: + f.write(content) + """ + return self._data.download_attachment(entity_id, record_id, field_name) + + @traced(name="entity_download_attachment", run_type="uipath") + async def download_attachment_async( + self, entity_id: str, record_id: str, field_name: str + ) -> bytes: + """Asynchronously download a file attached to a record. + + Args: + entity_id (str): The unique identifier of the entity. + record_id (str): The unique identifier of the record containing + the attachment. + field_name (str): Name of the File-type field on the entity. + + Returns: + bytes: The raw file content. + + Examples: + Save the downloaded bytes to disk:: + + content = await entities_service.download_attachment_async( + "Customers", "rec-1", "Contract" + ) + with open("downloaded.pdf", "wb") as f: + f.write(content) + """ + return await self._data.download_attachment_async( + entity_id, record_id, field_name ) - def _update_batch_spec( - self, entity_key: str, records: List[EntityRecord] - ) -> RequestSpec: - return RequestSpec( - method="POST", - endpoint=Endpoint( - f"datafabric_/api/EntityService/entity/{entity_key}/update-batch" - ), - json=[record.model_dump(by_alias=True) for record in records], + @traced(name="entity_delete_attachment", run_type="uipath") + def delete_attachment( + self, + entity_id: str, + record_id: str, + field_name: str, + expansion_level: Optional[int] = None, + ) -> Dict[str, Any]: + """Remove the file attached to a File-type field on a record. + + Args: + entity_id (str): The unique identifier of the entity. + record_id (str): The unique identifier of the record whose + attachment is being removed. + field_name (str): Name of the File-type field on the entity. + expansion_level (Optional[int]): Optional FK expansion depth in + the response (``0`` means no expansion). + + Returns: + Dict[str, Any]: The decoded JSON response (typically the updated + record), or an empty dict when the response has no body. + + Examples: + Clear an attachment:: + + entities_service.delete_attachment( + "Customers", "rec-1", "Contract" + ) + """ + return self._data.delete_attachment( + entity_id, record_id, field_name, expansion_level=expansion_level ) - def _delete_batch_spec(self, entity_key: str, record_ids: List[str]) -> RequestSpec: - return RequestSpec( - method="POST", - endpoint=Endpoint( - f"datafabric_/api/EntityService/entity/{entity_key}/delete-batch" - ), - json=record_ids, + @traced(name="entity_delete_attachment", run_type="uipath") + async def delete_attachment_async( + self, + entity_id: str, + record_id: str, + field_name: str, + expansion_level: Optional[int] = None, + ) -> Dict[str, Any]: + """Asynchronously remove the file attached to a File-type field. + + Args: + entity_id (str): The unique identifier of the entity. + record_id (str): The unique identifier of the record whose + attachment is being removed. + field_name (str): Name of the File-type field on the entity. + expansion_level (Optional[int]): Optional FK expansion depth. + + Returns: + Dict[str, Any]: The decoded JSON response. + + Examples: + Clear an attachment:: + + await entities_service.delete_attachment_async( + "Customers", "rec-1", "Contract" + ) + """ + return await self._data.delete_attachment_async( + entity_id, record_id, field_name, expansion_level=expansion_level ) - def _validate_sql_query(self, sql_query: str) -> None: - query = sql_query.strip().rstrip(";").strip() - if not query: - raise ValueError("SQL query cannot be empty.") + @traced(name="entity_import_records", run_type="uipath") + def import_records( + self, + entity_id: str, + file: Optional[FileContent] = None, + file_path: Optional[str] = None, + ) -> EntityImportRecordsResponse: + """Bulk-import records into an entity from a CSV file. - statements = sqlparse.parse(query) - if len(statements) != 1 or not statements[0].tokens: - raise ValueError("Only a single SELECT statement is allowed.") + Provide exactly one of ``file`` (raw bytes) or ``file_path`` (path on + disk). - stmt = statements[0] - stmt_type = stmt.get_type() + Args: + entity_id (str): The unique identifier of the entity. + file (Optional[FileContent]): Raw bytes of a CSV file. Mutually + exclusive with ``file_path``. + file_path (Optional[str]): Path to a local CSV file. Mutually + exclusive with ``file``. - if stmt_type != "SELECT": - raise ValueError("Only SELECT statements are allowed.") + Returns: + EntityImportRecordsResponse: Reports the total rows in the file, + the number successfully inserted, and an optional + ``error_file_link`` pointing to a CSV listing rows that + failed validation. - keywords = set() - for token in stmt.flatten(): - if token.ttype in Keyword: - keywords.add(token.normalized) + Examples: + Import from a path on disk:: - for kw in _FORBIDDEN_DML: - if kw in keywords: - raise ValueError(f"SQL keyword '{kw}' is not allowed.") + result = entities_service.import_records( + "Customers", file_path="./customers.csv" + ) + print( + f"Inserted {result.inserted_records} of " + f"{result.total_records} rows" + ) + if result.error_file_link: + print(f"Errors: {result.error_file_link}") + """ + return self._data.import_records(entity_id, file=file, file_path=file_path) + + @traced(name="entity_import_records", run_type="uipath") + async def import_records_async( + self, + entity_id: str, + file: Optional[FileContent] = None, + file_path: Optional[str] = None, + ) -> EntityImportRecordsResponse: + """Asynchronously bulk-import records into an entity from a CSV file. - for kw in _FORBIDDEN_DDL: - if kw in keywords: - raise ValueError(f"SQL keyword '{kw}' is not allowed.") + Provide exactly one of ``file`` (raw bytes) or ``file_path`` (path on + disk). - for kw in _DISALLOWED_KEYWORDS: - if kw in keywords: - raise ValueError( - f"SQL construct '{kw}' is not allowed in entity queries." + Args: + entity_id (str): The unique identifier of the entity. + file (Optional[FileContent]): Raw bytes of a CSV file. + file_path (Optional[str]): Path to a local CSV file. + + Returns: + EntityImportRecordsResponse: Reports the total, inserted, and + ``error_file_link`` for failed rows. + + Examples: + Import from a path on disk:: + + result = await entities_service.import_records_async( + "Customers", file_path="./customers.csv" + ) + print( + f"Inserted {result.inserted_records} of " + f"{result.total_records} rows" ) + """ + return await self._data.import_records_async( + entity_id, file=file, file_path=file_path + ) + + # ------------------------------------------------------------------ + # Public helper retained for backward compatibility — tests call this + # ------------------------------------------------------------------ + + def validate_entity_batch( + self, + batch_response: Response, + schema: Optional[Type[Any]] = None, + ) -> EntityRecordsBatchResponse: + """Parse a batch response, optionally validating success records against ``schema``. + + Failure records are returned as :class:`FailureRecord` instances and + are not validated against the user schema. + """ + return self._data.validate_entity_batch(batch_response, schema=schema) - if self._has_subquery(stmt): - raise ValueError("Subqueries are not allowed.") - - has_where = any(isinstance(t, Where) for t in stmt.tokens) - has_limit = "LIMIT" in keywords - if not has_where and not has_limit: - raise ValueError("Queries without WHERE must include a LIMIT clause.") - - projection = self._projection_tokens(stmt) - has_wildcard = any(t.ttype is Wildcard for t in projection) - if has_wildcard and not has_where: - raise ValueError("SELECT * without filtering is not allowed.") - if not has_where and self._projection_column_count(projection) > 4: - raise ValueError( - "Selecting more than 4 columns without filtering is not allowed." + # ------------------------------------------------------------------ + # Cross-cutting — entity-set resolution for agent overrides + # ------------------------------------------------------------------ + + @traced(name="resolve_entity_set", run_type="uipath") + def resolve_entity_set( + self, + items: List[DataFabricEntityItem], + ) -> EntitySetResolution: + """Resolve an agent entity set, applying resource overwrites.""" + plan = create_resolution_plan( + items, + _resource_overwrites.get() or {}, + lambda folder_path: ( + self._folders_service.retrieve_key(folder_path=folder_path) + if self._folders_service is not None + else None + ), + ) + entities = fetch_resolved_entities( + plan, + self.retrieve, + self.retrieve_by_name, + logger, + ) + resolution_service: EntitiesService = build_resolution_service( # type: ignore[assignment] + config=self._config, + execution_context=self._execution_context, + folders_service=self._folders_service, + plan=plan, + service_factory=EntitiesService, + ) + return EntitySetResolution( + entities=entities, + entities_service=resolution_service, + ) + + @traced(name="resolve_entity_set", run_type="uipath") + async def resolve_entity_set_async( + self, + items: List[DataFabricEntityItem], + ) -> EntitySetResolution: + """Resolve an agent entity set, applying resource overwrites.""" + + async def _resolve_folder_path(folder_path: str) -> Optional[str]: + if self._folders_service is None: + return None + return await self._folders_service.retrieve_key_async( + folder_path=folder_path ) - @staticmethod - def _has_subquery(stmt: sqlparse.sql.Statement) -> bool: - """Recursively walk the AST looking for SELECT inside parentheses.""" - - def _walk(token: sqlparse.sql.Token) -> bool: - if isinstance(token, Parenthesis): - for child in token.flatten(): - if child.ttype is DML and child.normalized == "SELECT": - return True - if hasattr(token, "tokens"): - for child in token.tokens: - if _walk(child): - return True - return False - - for token in stmt.tokens: - if _walk(token): - return True - return False - - @staticmethod - def _projection_tokens( - stmt: sqlparse.sql.Statement, - ) -> list[sqlparse.sql.Token]: - """Extract tokens between the first SELECT and FROM.""" - tokens: list[sqlparse.sql.Token] = [] - collecting = False - for token in stmt.flatten(): - if token.ttype is DML and token.normalized == "SELECT": - collecting = True - continue - if token.ttype is Keyword and token.normalized == "FROM": - break - if collecting: - tokens.append(token) - return tokens - - @staticmethod - def _projection_column_count( - projection: list[sqlparse.sql.Token], - ) -> int: - text = "".join(t.value for t in projection).strip() - if not text: - return 0 - return len([part for part in text.split(",") if part.strip()]) + plan = await create_resolution_plan_async( + items, + _resource_overwrites.get() or {}, + _resolve_folder_path, + ) + entities = await fetch_resolved_entities_async( + plan, + self.retrieve_async, + self.retrieve_by_name_async, + logger, + ) + resolution_service: EntitiesService = build_resolution_service( # type: ignore[assignment] + config=self._config, + execution_context=self._execution_context, + folders_service=self._folders_service, + plan=plan, + service_factory=EntitiesService, + ) + return EntitySetResolution( + entities=entities, + entities_service=resolution_service, + ) + + +# Resolve the forward reference to EntitiesService in EntitySetResolution. +# The model uses TYPE_CHECKING to avoid circular imports in entities.py, +# so we must rebuild it here where EntitiesService is fully defined. +EntitySetResolution.model_rebuild() diff --git a/packages/uipath-platform/src/uipath/platform/entities/_entity_data_service.py b/packages/uipath-platform/src/uipath/platform/entities/_entity_data_service.py new file mode 100644 index 000000000..4dc09855d --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/entities/_entity_data_service.py @@ -0,0 +1,1416 @@ +"""Data-side operations for the Data Fabric entities surface. + +Handles record CRUD (single and batch), structured queries, attachments, +choice-set value lookup, bulk import, and federated SQL queries. Schema +definitions are managed by :class:`EntitySchemaService` and exposed alongside +data operations through :class:`EntitiesService`. +""" + +import json as json_module +import logging +from contextlib import nullcontext +from pathlib import Path +from typing import Any, Dict, List, Optional, Type + +import sqlparse +from httpx import HTTPStatusError, Response +from pydantic import BaseModel +from sqlparse.sql import Function, Identifier, IdentifierList, Parenthesis, Where +from sqlparse.tokens import DML, Keyword, Whitespace, Wildcard + +from ..common._base_service import BaseService +from ..common._config import UiPathApiConfig +from ..common._execution_context import UiPathExecutionContext +from ..common._models import Endpoint, RequestSpec +from ..errors._enriched_exception import EnrichedException +from ..orchestrator._folder_service import FolderService +from ._entity_resolution import RoutingStrategy, create_routing_strategy +from .entities import ( + AggregateRow, + ChoiceSetValue, + EntityAggregate, + EntityBinning, + EntityImportRecordsResponse, + EntityJoin, + EntityQueryFilterGroup, + EntityQuerySortOption, + EntityRecord, + EntityRecordsBatchResponse, + EntityRecordsListResponse, + QueryRoutingOverrideContext, + RetrieveEntityRecordsResponse, +) + +logger = logging.getLogger(__name__) + +FileContent = bytes | bytearray | memoryview +"""Acceptable raw bytes types for attachment and CSV uploads.""" + +_FORBIDDEN_DML = {"INSERT", "UPDATE", "DELETE", "MERGE", "REPLACE"} +_FORBIDDEN_DDL = {"DROP", "ALTER", "CREATE", "TRUNCATE"} +_DISALLOWED_KEYWORDS = [ + "WITH", + "UNION", + "INTERSECT", + "EXCEPT", + "OVER", + "ROLLUP", + "CUBE", + "GROUPING", + "PARTITION", +] +_AGGREGATE_FUNCTIONS = ("COUNT", "SUM", "AVG", "MIN", "MAX") + + +class EntityDataService(BaseService): + """HTTP service for entity-record and attachment operations. + + Backend target: ``datafabric_/api/EntityService/...`` plus + ``datafabric_/api/Attachment/...`` for file attachments, and + ``datafabric_/api/v1/query/execute`` for federated SQL queries. + + !!! warning "Preview Feature" + This service is currently experimental. Behavior and parameters are + subject to change in future versions. + """ + + def __init__( + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + folders_service: Optional[FolderService] = None, + routing_strategy: Optional[RoutingStrategy] = None, + folders_map: Optional[Dict[str, str]] = None, + entity_name_overrides: Optional[Dict[str, str]] = None, + routing_context: Optional[QueryRoutingOverrideContext] = None, + ) -> None: + """Initialise the data service. + + Either pass a pre-built ``routing_strategy`` (the facade does this so + both services share one) or supply the inputs and let this service + construct its own. + """ + super().__init__(config=config, execution_context=execution_context) + self._folders_service = folders_service + self._routing_strategy: RoutingStrategy = ( + routing_strategy + if routing_strategy is not None + else create_routing_strategy( + folders_map=folders_map, + effective_entity_names=entity_name_overrides, + routing_context=routing_context, + folders_service=folders_service, + ) + ) + + # ------------------------------------------------------------------ + # Choice-set value lookup + # ------------------------------------------------------------------ + + def get_choiceset_values( + self, + choiceset_id: str, + start: Optional[int] = None, + limit: Optional[int] = None, + ) -> List[ChoiceSetValue]: + """Internal implementation; see :meth:`EntitiesService.get_choiceset_values`.""" + spec = self._get_choiceset_values_spec(choiceset_id, start=start, limit=limit) + response = self.request( + spec.method, spec.endpoint, params=spec.params, json=spec.json + ) + return self._parse_choiceset_values(response) + + async def get_choiceset_values_async( + self, + choiceset_id: str, + start: Optional[int] = None, + limit: Optional[int] = None, + ) -> List[ChoiceSetValue]: + """Async variant of :meth:`get_choiceset_values`.""" + spec = self._get_choiceset_values_spec(choiceset_id, start=start, limit=limit) + response = await self.request_async( + spec.method, spec.endpoint, params=spec.params, json=spec.json + ) + return self._parse_choiceset_values(response) + + # ------------------------------------------------------------------ + # List records (multi-record read with OData filters) + # ------------------------------------------------------------------ + + def list_records( + self, + entity_key: str, + schema: Optional[Type[Any]] = None, + start: Optional[int] = None, + limit: Optional[int] = None, + expansion_level: Optional[int] = None, + filter: Optional[str] = None, + orderby: Optional[str] = None, + select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, + ) -> EntityRecordsListResponse: + """Internal implementation; see :meth:`EntitiesService.list_records`.""" + spec = self._list_records_spec( + entity_key, + start=start, + limit=limit, + expansion_level=expansion_level, + filter=filter, + orderby=orderby, + select=select, + expand=expand, + ) + response = self.request(spec.method, spec.endpoint, params=spec.params) + return self._build_records_list_response(response, schema, start, limit) + + async def list_records_async( + self, + entity_key: str, + schema: Optional[Type[Any]] = None, + start: Optional[int] = None, + limit: Optional[int] = None, + expansion_level: Optional[int] = None, + filter: Optional[str] = None, + orderby: Optional[str] = None, + select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, + ) -> EntityRecordsListResponse: + """Async variant of :meth:`list_records`.""" + spec = self._list_records_spec( + entity_key, + start=start, + limit=limit, + expansion_level=expansion_level, + filter=filter, + orderby=orderby, + select=select, + expand=expand, + ) + response = await self.request_async( + spec.method, spec.endpoint, params=spec.params + ) + return self._build_records_list_response(response, schema, start, limit) + + # ------------------------------------------------------------------ + # Single-record operations (fire trigger events; batch versions don't) + # ------------------------------------------------------------------ + + def insert_record( + self, + entity_key: str, + data: Any, + expansion_level: Optional[int] = None, + ) -> EntityRecord: + """Internal implementation; see :meth:`EntitiesService.insert_record`.""" + spec = self._insert_record_spec(entity_key, data, expansion_level) + response = self.request( + spec.method, spec.endpoint, params=spec.params, json=spec.json + ) + return EntityRecord.model_validate(response.json()) + + async def insert_record_async( + self, + entity_key: str, + data: Any, + expansion_level: Optional[int] = None, + ) -> EntityRecord: + """Async variant of :meth:`insert_record`.""" + spec = self._insert_record_spec(entity_key, data, expansion_level) + response = await self.request_async( + spec.method, spec.endpoint, params=spec.params, json=spec.json + ) + return EntityRecord.model_validate(response.json()) + + def get_record( + self, + entity_key: str, + record_id: str, + expansion_level: Optional[int] = None, + ) -> EntityRecord: + """Fetch a single record by its id.""" + spec = self._get_record_spec(entity_key, record_id, expansion_level) + response = self.request(spec.method, spec.endpoint, params=spec.params) + return EntityRecord.model_validate(response.json()) + + async def get_record_async( + self, + entity_key: str, + record_id: str, + expansion_level: Optional[int] = None, + ) -> EntityRecord: + """Async variant of :meth:`get_record`.""" + spec = self._get_record_spec(entity_key, record_id, expansion_level) + response = await self.request_async( + spec.method, spec.endpoint, params=spec.params + ) + return EntityRecord.model_validate(response.json()) + + def update_record( + self, + entity_key: str, + record_id: str, + data: Any, + expansion_level: Optional[int] = None, + ) -> EntityRecord: + """Internal implementation; see :meth:`EntitiesService.update_record`.""" + spec = self._update_record_spec(entity_key, record_id, data, expansion_level) + response = self.request( + spec.method, spec.endpoint, params=spec.params, json=spec.json + ) + return EntityRecord.model_validate(response.json()) + + async def update_record_async( + self, + entity_key: str, + record_id: str, + data: Any, + expansion_level: Optional[int] = None, + ) -> EntityRecord: + """Async variant of :meth:`update_record`.""" + spec = self._update_record_spec(entity_key, record_id, data, expansion_level) + response = await self.request_async( + spec.method, spec.endpoint, params=spec.params, json=spec.json + ) + return EntityRecord.model_validate(response.json()) + + def delete_record(self, entity_key: str, record_id: str) -> None: + """Delete a single record by id.""" + spec = self._delete_record_spec(entity_key, record_id) + self.request(spec.method, spec.endpoint) + + async def delete_record_async(self, entity_key: str, record_id: str) -> None: + """Async variant of :meth:`delete_record`.""" + spec = self._delete_record_spec(entity_key, record_id) + await self.request_async(spec.method, spec.endpoint) + + # ------------------------------------------------------------------ + # Batch record operations + # ------------------------------------------------------------------ + + def insert_records( + self, + entity_key: str, + records: List[Any], + schema: Optional[Type[Any]] = None, + expansion_level: Optional[int] = None, + fail_on_first: Optional[bool] = None, + ) -> EntityRecordsBatchResponse: + """Internal implementation; see :meth:`EntitiesService.insert_records`.""" + spec = self._insert_batch_spec( + entity_key, + records, + expansion_level=expansion_level, + fail_on_first=fail_on_first, + ) + response = self._request_or_extract_batch( + sync_call=lambda: self.request( + spec.method, spec.endpoint, params=spec.params, json=spec.json + ) + ) + if isinstance(response, EntityRecordsBatchResponse): + return response + return self.validate_entity_batch(response, schema) + + async def insert_records_async( + self, + entity_key: str, + records: List[Any], + schema: Optional[Type[Any]] = None, + expansion_level: Optional[int] = None, + fail_on_first: Optional[bool] = None, + ) -> EntityRecordsBatchResponse: + """Async variant of :meth:`insert_records`.""" + spec = self._insert_batch_spec( + entity_key, + records, + expansion_level=expansion_level, + fail_on_first=fail_on_first, + ) + + async def _send_batch() -> Response: + return await self.request_async( + spec.method, spec.endpoint, params=spec.params, json=spec.json + ) + + result = await self._request_or_extract_batch_async(_send_batch) + if isinstance(result, EntityRecordsBatchResponse): + return result + return self.validate_entity_batch(result, schema) + + def update_records( + self, + entity_key: str, + records: List[Any], + schema: Optional[Type[Any]] = None, + expansion_level: Optional[int] = None, + fail_on_first: Optional[bool] = None, + ) -> EntityRecordsBatchResponse: + """Internal implementation; see :meth:`EntitiesService.update_records`.""" + normalized = [self._record_to_dict(record) for record in records] + if schema is not None: + for record in normalized: + EntityRecord.from_data(data=record, model=schema) + + spec = self._update_batch_spec( + entity_key, + normalized, + expansion_level=expansion_level, + fail_on_first=fail_on_first, + ) + response = self._request_or_extract_batch( + sync_call=lambda: self.request( + spec.method, spec.endpoint, params=spec.params, json=spec.json + ) + ) + if isinstance(response, EntityRecordsBatchResponse): + return response + return self.validate_entity_batch(response, schema) + + async def update_records_async( + self, + entity_key: str, + records: List[Any], + schema: Optional[Type[Any]] = None, + expansion_level: Optional[int] = None, + fail_on_first: Optional[bool] = None, + ) -> EntityRecordsBatchResponse: + """Async variant of :meth:`update_records`.""" + normalized = [self._record_to_dict(record) for record in records] + if schema is not None: + for record in normalized: + EntityRecord.from_data(data=record, model=schema) + + spec = self._update_batch_spec( + entity_key, + normalized, + expansion_level=expansion_level, + fail_on_first=fail_on_first, + ) + + async def _send_batch() -> Response: + return await self.request_async( + spec.method, spec.endpoint, params=spec.params, json=spec.json + ) + + result = await self._request_or_extract_batch_async(_send_batch) + if isinstance(result, EntityRecordsBatchResponse): + return result + return self.validate_entity_batch(result, schema) + + def delete_records( + self, + entity_key: str, + record_ids: List[str], + fail_on_first: Optional[bool] = None, + ) -> EntityRecordsBatchResponse: + """Delete multiple records by id in a single batch.""" + spec = self._delete_batch_spec( + entity_key, record_ids, fail_on_first=fail_on_first + ) + result = self._request_or_extract_batch( + sync_call=lambda: self.request( + spec.method, spec.endpoint, params=spec.params, json=spec.json + ) + ) + if isinstance(result, EntityRecordsBatchResponse): + return result + return EntityRecordsBatchResponse.model_validate(result.json()) + + async def delete_records_async( + self, + entity_key: str, + record_ids: List[str], + fail_on_first: Optional[bool] = None, + ) -> EntityRecordsBatchResponse: + """Async variant of :meth:`delete_records`.""" + spec = self._delete_batch_spec( + entity_key, record_ids, fail_on_first=fail_on_first + ) + + async def _send_batch() -> Response: + return await self.request_async( + spec.method, spec.endpoint, params=spec.params, json=spec.json + ) + + result = await self._request_or_extract_batch_async(_send_batch) + if isinstance(result, EntityRecordsBatchResponse): + return result + return EntityRecordsBatchResponse.model_validate(result.json()) + + # ------------------------------------------------------------------ + # Structured query (POST /entity/{id}/query) + # ------------------------------------------------------------------ + + def retrieve_records( + self, + entity_key: str, + filter_group: Optional[EntityQueryFilterGroup] = None, + sort_options: Optional[List[EntityQuerySortOption]] = None, + selected_fields: Optional[List[str]] = None, + expansions: Optional[List[Any]] = None, + expansion_level: Optional[int] = None, + aggregates: Optional[List[EntityAggregate]] = None, + group_by: Optional[List[str]] = None, + joins: Optional[List[EntityJoin]] = None, + binnings: Optional[List[EntityBinning]] = None, + start: Optional[int] = None, + limit: Optional[int] = None, + ) -> RetrieveEntityRecordsResponse: + """Internal implementation; see :meth:`EntitiesService.retrieve_records`.""" + spec = self._retrieve_records_spec( + entity_key, + filter_group=filter_group, + sort_options=sort_options, + selected_fields=selected_fields, + expansions=expansions, + expansion_level=expansion_level, + aggregates=aggregates, + group_by=group_by, + joins=joins, + binnings=binnings, + start=start, + limit=limit, + ) + response = self.request( + spec.method, spec.endpoint, params=spec.params, json=spec.json + ) + return self._parse_query_response(response, start=start, limit=limit) + + async def retrieve_records_async( + self, + entity_key: str, + filter_group: Optional[EntityQueryFilterGroup] = None, + sort_options: Optional[List[EntityQuerySortOption]] = None, + selected_fields: Optional[List[str]] = None, + expansions: Optional[List[Any]] = None, + expansion_level: Optional[int] = None, + aggregates: Optional[List[EntityAggregate]] = None, + group_by: Optional[List[str]] = None, + joins: Optional[List[EntityJoin]] = None, + binnings: Optional[List[EntityBinning]] = None, + start: Optional[int] = None, + limit: Optional[int] = None, + ) -> RetrieveEntityRecordsResponse: + """Async variant of :meth:`retrieve_records`.""" + spec = self._retrieve_records_spec( + entity_key, + filter_group=filter_group, + sort_options=sort_options, + selected_fields=selected_fields, + expansions=expansions, + expansion_level=expansion_level, + aggregates=aggregates, + group_by=group_by, + joins=joins, + binnings=binnings, + start=start, + limit=limit, + ) + response = await self.request_async( + spec.method, spec.endpoint, params=spec.params, json=spec.json + ) + return self._parse_query_response(response, start=start, limit=limit) + + # ------------------------------------------------------------------ + # Federated SQL query + # ------------------------------------------------------------------ + + def query_entity_records( + self, + sql_query: str, + relationships_as_scalar: bool = False, + ) -> List[Dict[str, Any]]: + """Internal implementation; see :meth:`EntitiesService.query_entity_records`.""" + return self._query_entities_for_records(sql_query, relationships_as_scalar) + + async def query_entity_records_async( + self, + sql_query: str, + relationships_as_scalar: bool = False, + ) -> List[Dict[str, Any]]: + """Async variant of :meth:`query_entity_records`.""" + return await self._query_entities_for_records_async( + sql_query, relationships_as_scalar + ) + + # ------------------------------------------------------------------ + # Attachments + # ------------------------------------------------------------------ + + def upload_attachment( + self, + entity_id: str, + record_id: str, + field_name: str, + file: Optional[FileContent] = None, + file_path: Optional[str] = None, + expansion_level: Optional[int] = None, + ) -> Dict[str, Any]: + """Internal implementation; see :meth:`EntitiesService.upload_attachment`.""" + spec = self._attachment_endpoint( + entity_id, record_id, field_name, expansion_level + ) + with self._open_file(file, file_path) as handle: + response = self.request( + "POST", + spec.endpoint, + params=spec.params, + files={"file": handle}, + ) + return response.json() if response.content else {} + + async def upload_attachment_async( + self, + entity_id: str, + record_id: str, + field_name: str, + file: Optional[FileContent] = None, + file_path: Optional[str] = None, + expansion_level: Optional[int] = None, + ) -> Dict[str, Any]: + """Async variant of :meth:`upload_attachment`.""" + spec = self._attachment_endpoint( + entity_id, record_id, field_name, expansion_level + ) + with self._open_file(file, file_path) as handle: + response = await self.request_async( + "POST", + spec.endpoint, + params=spec.params, + files={"file": handle}, + ) + return response.json() if response.content else {} + + def download_attachment( + self, entity_id: str, record_id: str, field_name: str + ) -> bytes: + """Internal implementation; see :meth:`EntitiesService.download_attachment`.""" + spec = self._attachment_endpoint(entity_id, record_id, field_name) + response = self.request("GET", spec.endpoint) + return response.content + + async def download_attachment_async( + self, entity_id: str, record_id: str, field_name: str + ) -> bytes: + """Async variant of :meth:`download_attachment`.""" + spec = self._attachment_endpoint(entity_id, record_id, field_name) + response = await self.request_async("GET", spec.endpoint) + return response.content + + def delete_attachment( + self, + entity_id: str, + record_id: str, + field_name: str, + expansion_level: Optional[int] = None, + ) -> Dict[str, Any]: + """Internal implementation; see :meth:`EntitiesService.delete_attachment`.""" + spec = self._attachment_endpoint( + entity_id, record_id, field_name, expansion_level + ) + response = self.request("DELETE", spec.endpoint, params=spec.params) + return response.json() if response.content else {} + + async def delete_attachment_async( + self, + entity_id: str, + record_id: str, + field_name: str, + expansion_level: Optional[int] = None, + ) -> Dict[str, Any]: + """Async variant of :meth:`delete_attachment`.""" + spec = self._attachment_endpoint( + entity_id, record_id, field_name, expansion_level + ) + response = await self.request_async("DELETE", spec.endpoint, params=spec.params) + return response.json() if response.content else {} + + # ------------------------------------------------------------------ + # Bulk import + # ------------------------------------------------------------------ + + def import_records( + self, + entity_id: str, + file: Optional[FileContent] = None, + file_path: Optional[str] = None, + ) -> EntityImportRecordsResponse: + """Internal implementation; see :meth:`EntitiesService.import_records`.""" + spec = self._import_records_spec(entity_id) + with self._open_file(file, file_path) as handle: + response = self.request(spec.method, spec.endpoint, files={"file": handle}) + return EntityImportRecordsResponse.model_validate(response.json() or {}) + + async def import_records_async( + self, + entity_id: str, + file: Optional[FileContent] = None, + file_path: Optional[str] = None, + ) -> EntityImportRecordsResponse: + """Async variant of :meth:`import_records`.""" + spec = self._import_records_spec(entity_id) + with self._open_file(file, file_path) as handle: + response = await self.request_async( + spec.method, spec.endpoint, files={"file": handle} + ) + return EntityImportRecordsResponse.model_validate(response.json() or {}) + + # ------------------------------------------------------------------ + # Public helper for batch response validation + # ------------------------------------------------------------------ + + def validate_entity_batch( + self, + batch_response: Response, + schema: Optional[Type[Any]] = None, + ) -> EntityRecordsBatchResponse: + """Internal implementation; see :meth:`EntitiesService.validate_entity_batch`.""" + parsed = EntityRecordsBatchResponse.model_validate(batch_response.json()) + + validated_successful_records = [] + for successful_record in parsed.success_records: + data = successful_record.model_dump(by_alias=True) + if data.get("Id") is not None: + validated_successful_records.append( + EntityRecord.from_data(data=data, model=schema) + ) + + return EntityRecordsBatchResponse( + success_records=validated_successful_records, + failure_records=parsed.failure_records, + ) + + # ------------------------------------------------------------------ + # Internal helpers — request specs + # ------------------------------------------------------------------ + + def _query_entities_for_records( + self, sql_query: str, relationships_as_scalar: bool = False + ) -> List[Dict[str, Any]]: + """Synchronously run a validated SQL query through the federated query engine.""" + self._validate_sql_query(sql_query) + routing_context = self._routing_strategy.resolve() + spec = self._query_entity_records_spec( + sql_query, routing_context, relationships_as_scalar + ) + response = self.request(spec.method, spec.endpoint, json=spec.json) + return response.json().get("results", []) + + async def _query_entities_for_records_async( + self, sql_query: str, relationships_as_scalar: bool = False + ) -> List[Dict[str, Any]]: + """Asynchronously run a validated SQL query through the federated query engine.""" + self._validate_sql_query(sql_query) + routing_context = await self._routing_strategy.resolve_async() + spec = self._query_entity_records_spec( + sql_query, routing_context, relationships_as_scalar + ) + response = await self.request_async(spec.method, spec.endpoint, json=spec.json) + return response.json().get("results", []) + + @staticmethod + def _list_records_spec( + entity_key: str, + start: Optional[int] = None, + limit: Optional[int] = None, + expansion_level: Optional[int] = None, + filter: Optional[str] = None, + orderby: Optional[str] = None, + select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, + ) -> RequestSpec: + """Build the GET spec for the multi-record read endpoint.""" + params: Dict[str, Any] = {} + if start is not None: + params["start"] = start + if limit is not None: + params["limit"] = limit + if expansion_level is not None: + params["expansionLevel"] = expansion_level + if filter is not None: + params["$filter"] = filter + if orderby is not None: + params["$orderby"] = orderby + if select: + params["$select"] = ",".join(select) + if expand: + params["$expand"] = ",".join(expand) + return RequestSpec( + method="GET", + endpoint=Endpoint( + f"datafabric_/api/EntityService/entity/{entity_key}/read" + ), + params=params, + ) + + @staticmethod + def _insert_record_spec( + entity_key: str, + data: Any, + expansion_level: Optional[int] = None, + ) -> RequestSpec: + """Build the POST spec for inserting a single record.""" + params: Dict[str, Any] = {} + if expansion_level is not None: + params["expansionLevel"] = expansion_level + return RequestSpec( + method="POST", + endpoint=Endpoint( + f"datafabric_/api/EntityService/entity/{entity_key}/insert" + ), + params=params, + json=EntityDataService._record_to_dict(data), + ) + + @staticmethod + def _get_record_spec( + entity_key: str, + record_id: str, + expansion_level: Optional[int] = None, + ) -> RequestSpec: + """Build the GET spec for fetching a single record by id.""" + params: Dict[str, Any] = {} + if expansion_level is not None: + params["expansionLevel"] = expansion_level + return RequestSpec( + method="GET", + endpoint=Endpoint( + f"datafabric_/api/EntityService/entity/{entity_key}/read/{record_id}" + ), + params=params, + ) + + @staticmethod + def _update_record_spec( + entity_key: str, + record_id: str, + data: Any, + expansion_level: Optional[int] = None, + ) -> RequestSpec: + """Build the POST spec for updating a single record by id.""" + params: Dict[str, Any] = {} + if expansion_level is not None: + params["expansionLevel"] = expansion_level + return RequestSpec( + method="POST", + endpoint=Endpoint( + f"datafabric_/api/EntityService/entity/{entity_key}/update/{record_id}" + ), + params=params, + json=EntityDataService._record_to_dict(data), + ) + + @staticmethod + def _delete_record_spec(entity_key: str, record_id: str) -> RequestSpec: + """Build the DELETE spec for removing a single record by id.""" + return RequestSpec( + method="DELETE", + endpoint=Endpoint( + f"datafabric_/api/EntityService/entity/{entity_key}/delete/{record_id}" + ), + ) + + @staticmethod + def _insert_batch_spec( + entity_key: str, + records: List[Any], + expansion_level: Optional[int] = None, + fail_on_first: Optional[bool] = None, + ) -> RequestSpec: + """Build the POST spec for the batch-insert endpoint.""" + params = EntityDataService._batch_params( + expansion_level=expansion_level, fail_on_first=fail_on_first + ) + return RequestSpec( + method="POST", + endpoint=Endpoint( + f"datafabric_/api/EntityService/entity/{entity_key}/insert-batch" + ), + params=params, + json=[EntityDataService._record_to_dict(record) for record in records], + ) + + @staticmethod + def _update_batch_spec( + entity_key: str, + records: List[Dict[str, Any]], + expansion_level: Optional[int] = None, + fail_on_first: Optional[bool] = None, + ) -> RequestSpec: + """Build the POST spec for the batch-update endpoint.""" + params = EntityDataService._batch_params( + expansion_level=expansion_level, fail_on_first=fail_on_first + ) + return RequestSpec( + method="POST", + endpoint=Endpoint( + f"datafabric_/api/EntityService/entity/{entity_key}/update-batch" + ), + params=params, + json=records, + ) + + @staticmethod + def _delete_batch_spec( + entity_key: str, + record_ids: List[str], + fail_on_first: Optional[bool] = None, + ) -> RequestSpec: + """Build the POST spec for the batch-delete endpoint.""" + params = EntityDataService._batch_params(fail_on_first=fail_on_first) + return RequestSpec( + method="POST", + endpoint=Endpoint( + f"datafabric_/api/EntityService/entity/{entity_key}/delete-batch" + ), + params=params, + json=record_ids, + ) + + @staticmethod + def _batch_params( + expansion_level: Optional[int] = None, + fail_on_first: Optional[bool] = None, + ) -> Dict[str, Any]: + """Build the optional URL params common to all batch endpoints.""" + params: Dict[str, Any] = {} + if expansion_level is not None: + params["expansionLevel"] = expansion_level + if fail_on_first is not None: + params["failOnFirst"] = "true" if fail_on_first else "false" + return params + + @staticmethod + def _retrieve_records_spec( + entity_key: str, + filter_group: Optional[EntityQueryFilterGroup] = None, + sort_options: Optional[List[EntityQuerySortOption]] = None, + selected_fields: Optional[List[str]] = None, + expansions: Optional[List[Any]] = None, + expansion_level: Optional[int] = None, + aggregates: Optional[List[Any]] = None, + group_by: Optional[List[str]] = None, + joins: Optional[List[EntityJoin]] = None, + binnings: Optional[List[EntityBinning]] = None, + start: Optional[int] = None, + limit: Optional[int] = None, + ) -> RequestSpec: + """Build the request spec for the structured-query endpoint. + + Filters, sorting, projection, expansions, aggregates, group-by, joins, + binnings, ``start``, and ``limit`` are placed in the JSON body; + ``expansionLevel`` is a URL query parameter. The V2 endpoint is used + only when ``binnings`` are supplied. + """ + body: Dict[str, Any] = {} + if filter_group is not None: + body["filterGroup"] = filter_group.model_dump( + by_alias=True, exclude_none=True + ) + if sort_options: + body["sortOptions"] = [ + opt.model_dump(by_alias=True, exclude_none=True) for opt in sort_options + ] + if selected_fields: + body["selectedFields"] = list(selected_fields) + if expansions: + body["expansions"] = [ + e.model_dump(by_alias=True, exclude_none=True) + if isinstance(e, BaseModel) + else e + for e in expansions + ] + if aggregates: + body["aggregates"] = [ + a.model_dump(by_alias=True, exclude_none=True) + if isinstance(a, BaseModel) + else a + for a in aggregates + ] + if group_by: + body["groupBy"] = list(group_by) + if joins: + body["joins"] = [ + j.model_dump(by_alias=True, exclude_none=True) for j in joins + ] + if binnings: + body["binnings"] = [ + b.model_dump(by_alias=True, exclude_none=True) for b in binnings + ] + if start is not None: + body["start"] = start + if limit is not None: + body["limit"] = limit + + params: Dict[str, Any] = {} + if expansion_level is not None: + params["expansionLevel"] = expansion_level + + if binnings: + endpoint = Endpoint( + f"datafabric_/api/v2/EntityService/entity/{entity_key}/query" + ) + else: + endpoint = Endpoint( + f"datafabric_/api/EntityService/entity/{entity_key}/query" + ) + + return RequestSpec( + method="POST", + endpoint=endpoint, + params=params, + json=body, + ) + + @staticmethod + def _query_entity_records_spec( + sql_query: str, + routing_context: Optional[QueryRoutingOverrideContext] = None, + relationships_as_scalar: bool = False, + ) -> RequestSpec: + """Build the POST spec for the federated SQL query endpoint.""" + body: Dict[str, Any] = {"query": sql_query} + if routing_context: + body["routingContext"] = routing_context.model_dump( + by_alias=True, exclude_none=True + ) + if relationships_as_scalar: + body["queryOptions"] = {"relationshipsAsScalar": True} + return RequestSpec( + method="POST", + endpoint=Endpoint("datafabric_/api/v1/query/execute"), + json=body, + ) + + @staticmethod + def _get_choiceset_values_spec( + choiceset_id: str, + start: Optional[int] = None, + limit: Optional[int] = None, + ) -> RequestSpec: + """Build the POST spec for the choice-set values endpoint.""" + params: Dict[str, Any] = {} + if start is not None: + params["start"] = start + if limit is not None: + params["limit"] = limit + return RequestSpec( + method="POST", + endpoint=Endpoint( + f"datafabric_/api/EntityService/entity/{choiceset_id}/query_expansion" + ), + params=params, + json={}, + ) + + @staticmethod + def _attachment_endpoint( + entity_id: str, + record_id: str, + field_name: str, + expansion_level: Optional[int] = None, + ) -> RequestSpec: + """Return the attachment endpoint plus any ``expansionLevel`` query param. + + The HTTP verb is supplied by the caller; only the URL and query + parameters depend on these arguments. + """ + params: Dict[str, Any] = {} + if expansion_level is not None: + params["expansionLevel"] = expansion_level + return RequestSpec( + method="POST", + endpoint=Endpoint( + f"datafabric_/api/Attachment/entity/{entity_id}/{record_id}/{field_name}" + ), + params=params, + ) + + @staticmethod + def _import_records_spec(entity_id: str) -> RequestSpec: + """Build the POST spec for the bulk-upload (CSV import) endpoint.""" + return RequestSpec( + method="POST", + endpoint=Endpoint( + f"datafabric_/api/EntityService/entity/{entity_id}/bulk-upload" + ), + ) + + @staticmethod + def _open_file(file: Optional[FileContent], file_path: Optional[str]) -> Any: + """Yield a file-like object from raw bytes or a path on disk. + + Exactly one of ``file`` and ``file_path`` must be supplied. + """ + if (file is None) == (file_path is None): + raise ValueError( + "Provide exactly one of `file` (bytes) or `file_path` (str path on disk)." + ) + if file_path is not None: + return open(Path(file_path), "rb") + return nullcontext(file) + + # ------------------------------------------------------------------ + # Internal helpers — response parsing and record normalisation + # ------------------------------------------------------------------ + + @staticmethod + def _record_to_dict(record: Any) -> Dict[str, Any]: + """Normalize an input record to a plain dict. + + Accepts dicts, Pydantic ``BaseModel`` (including :class:`EntityRecord`), + or any object exposing ``__dict__``. Explicit ``None`` values are + preserved so callers can clear fields by setting them to ``None`` on a + model instance — only unset fields (whose Pydantic default applies) are + dropped via ``exclude_unset=True``. + """ + if isinstance(record, dict): + return dict(record) + if isinstance(record, BaseModel): + return record.model_dump(by_alias=True, exclude_unset=True) + if hasattr(record, "__dict__"): + return {k: v for k, v in record.__dict__.items() if not k.startswith("_")} + raise TypeError( + f"Cannot convert record of type {type(record).__name__} to dict — " + "pass a dict, an EntityRecord, a Pydantic BaseModel, or an object with __dict__." + ) + + @staticmethod + def _build_records_list_response( + response: Response, + schema: Optional[Type[Any]], + start: Optional[int], + limit: Optional[int], + ) -> EntityRecordsListResponse: + """Build an :class:`EntityRecordsListResponse` from a list-records body.""" + body = response.json() or {} + records_data = body.get("value", []) + total_count = int( + body.get("totalRecordCount", body.get("totalCount", len(records_data))) or 0 + ) + records = [ + EntityRecord.from_data(data=record, model=schema) for record in records_data + ] + + next_cursor = body.get("nextCursor") + if limit is not None and limit > 0: + consumed = (start or 0) + len(records) + has_next_page = consumed < total_count + else: + has_next_page = bool(body.get("hasNextPage", False)) + + return EntityRecordsListResponse( + items=records, + total_count=total_count, + has_next_page=has_next_page, + next_cursor=next_cursor, + ) + + @staticmethod + def _parse_query_response( + response: Response, + start: Optional[int] = None, + limit: Optional[int] = None, + ) -> RetrieveEntityRecordsResponse: + """Parse a query response into :class:`RetrieveEntityRecordsResponse`. + + Rows that include an ``Id`` field are parsed as :class:`EntityRecord`; + rows that don't (aggregate / group-by / binning results) are parsed as + :class:`AggregateRow`. ``has_next_page`` is derived from + ``start + len(items) < total_count`` whenever ``limit`` is supplied; + ``next_cursor`` is populated only when the backend returns one, + otherwise the caller paginates by passing the next ``start``. + """ + body = response.json() or {} + items_raw = body.get("value", []) or [] + items: List[EntityRecord | AggregateRow] = [] + for raw in items_raw: + if isinstance(raw, dict) and isinstance(raw.get("Id"), str): + items.append(EntityRecord.from_data(data=raw)) + else: + items.append(AggregateRow.model_validate(raw)) + + total_count = int(body.get("totalRecordCount", body.get("totalCount", 0)) or 0) + + next_cursor: Optional[str] = body.get("nextCursor") + has_next_page = bool(body.get("hasNextPage", False)) + if next_cursor is None and limit is not None and limit > 0: + consumed = (start or 0) + len(items) + has_next_page = consumed < total_count + + return RetrieveEntityRecordsResponse( + items=items, + total_count=total_count, + has_next_page=has_next_page, + next_cursor=next_cursor, + ) + + @staticmethod + def _parse_choiceset_values(response: Response) -> List[ChoiceSetValue]: + """Decode and return the choice-set values from a query-expansion response.""" + data = response.json() + raw_values = data.get("jsonValue", "[]") + items = ( + json_module.loads(raw_values) if isinstance(raw_values, str) else raw_values + ) + return [ChoiceSetValue.model_validate(item) for item in items] + + # ------------------------------------------------------------------ + # Internal helpers — batch error recovery + # ------------------------------------------------------------------ + + def _request_or_extract_batch( + self, + sync_call: Any, + ) -> Response | EntityRecordsBatchResponse: + """Run a batch request and recover per-record failures from a 400 body. + + On HTTP 400 with a body that lists both ``successRecords`` and + ``failureRecords``, returns the parsed batch response instead of + raising. All other errors propagate. + """ + try: + return sync_call() + except EnrichedException as exc: + extracted = self._extract_batch_response_from_error(exc) + if extracted is not None: + return extracted + raise + + async def _request_or_extract_batch_async( + self, + async_call: Any, + ) -> Response | EntityRecordsBatchResponse: + """Async variant of :meth:`_request_or_extract_batch`.""" + try: + return await async_call() + except EnrichedException as exc: + extracted = self._extract_batch_response_from_error(exc) + if extracted is not None: + return extracted + raise + + @staticmethod + def _extract_batch_response_from_error( + exc: EnrichedException, + ) -> Optional[EntityRecordsBatchResponse]: + """Return a parsed batch response when the error body matches the per-record-failure shape. + + Recovery is intentionally narrow: only HTTP 400 with a JSON object + containing list-typed ``successRecords`` and ``failureRecords`` keys. + Returns ``None`` for any other status, body shape, or parse failure + so that the original error propagates. + """ + cause = exc.__cause__ + if not isinstance(cause, HTTPStatusError): + return None + if cause.response.status_code != 400: + return None + try: + data = cause.response.json() + except Exception: + return None + if not isinstance(data, dict): + return None + if not ( + isinstance(data.get("successRecords"), list) + and isinstance(data.get("failureRecords"), list) + ): + return None + try: + return EntityRecordsBatchResponse.model_validate(data) + except Exception: + return None + + # ------------------------------------------------------------------ + # Internal helpers — SQL validation (federated query path) + # ------------------------------------------------------------------ + + def _validate_sql_query(self, sql_query: str) -> None: + """Validate a SQL string for the federated query endpoint client-side.""" + query = sql_query.strip().rstrip(";").strip() + if not query: + raise ValueError("SQL query cannot be empty.") + + statements = sqlparse.parse(query) + if len(statements) != 1 or not statements[0].tokens: + raise ValueError("Only a single SELECT statement is allowed.") + + stmt = statements[0] + stmt_type = stmt.get_type() + + if stmt_type != "SELECT": + raise ValueError("Only SELECT statements are allowed.") + + keywords = set() + for token in stmt.flatten(): + if token.ttype in Keyword: + keywords.add(token.normalized) + + for kw in _FORBIDDEN_DML: + if kw in keywords: + raise ValueError(f"SQL keyword '{kw}' is not allowed.") + + for kw in _FORBIDDEN_DDL: + if kw in keywords: + raise ValueError(f"SQL keyword '{kw}' is not allowed.") + + for kw in _DISALLOWED_KEYWORDS: + if kw in keywords: + raise ValueError( + f"SQL construct '{kw}' is not allowed in entity queries." + ) + + if self._has_subquery(stmt): + raise ValueError("Subqueries are not allowed.") + + has_where = any(isinstance(t, Where) for t in stmt.tokens) + has_limit = "LIMIT" in keywords + has_from = "FROM" in keywords + + if not has_from: + raise ValueError("Queries must include a FROM clause.") + + projection = self._projection_tokens(stmt) + + if self._projection_has_count_star(projection): + raise ValueError( + "COUNT(*) is not supported. Use COUNT(column_name) instead." + ) + + has_aggregate = self._projection_has_aggregate(projection) + + if not has_where and not has_limit and not has_aggregate: + raise ValueError("Queries without WHERE must include a LIMIT clause.") + + has_bare_wildcard = self._projection_has_bare_wildcard(projection) + if has_bare_wildcard: + raise ValueError("SELECT * is not allowed. Specify column names instead.") + if not has_where and self._projection_column_count(projection) > 4: + raise ValueError( + "Selecting more than 4 columns without filtering is not allowed." + ) + + @staticmethod + def _projection_has_aggregate( + projection: List[sqlparse.sql.Token], + ) -> bool: + """Return ``True`` when the projection contains an aggregate function call.""" + + def _has_agg(token: sqlparse.sql.Token) -> bool: + if isinstance(token, Function): + return token.get_name().upper() in _AGGREGATE_FUNCTIONS + if isinstance(token, Identifier): + return any(_has_agg(child) for child in token.tokens) + return False + + for node in projection: + if _has_agg(node): + return True + if isinstance(node, IdentifierList): + if any(_has_agg(child) for child in node.tokens): + return True + return False + + @staticmethod + def _projection_has_count_star( + projection: List[sqlparse.sql.Token], + ) -> bool: + """Return ``True`` when the projection contains ``COUNT(*)``.""" + + def _is_count_star(func: Function) -> bool: + if func.get_name().upper() != "COUNT": + return False + return any(t.ttype is Wildcard for t in func.flatten()) + + def _has_count_star(token: sqlparse.sql.Token) -> bool: + if isinstance(token, Function): + return _is_count_star(token) + if isinstance(token, Identifier): + return any(_has_count_star(child) for child in token.tokens) + return False + + for node in projection: + if _has_count_star(node): + return True + if isinstance(node, IdentifierList): + if any(_has_count_star(child) for child in node.tokens): + return True + return False + + @staticmethod + def _projection_has_bare_wildcard( + projection: List[sqlparse.sql.Token], + ) -> bool: + """Return ``True`` for a bare ``*`` or qualified ``table.*`` outside a function.""" + + def _identifier_has_wildcard(ident: Identifier) -> bool: + return any(t.ttype is Wildcard for t in ident.tokens) + + for node in projection: + if node.ttype is Wildcard: + return True + if isinstance(node, Identifier) and _identifier_has_wildcard(node): + return True + if isinstance(node, IdentifierList): + for child in node.tokens: + if child.ttype is Wildcard: + return True + if isinstance(child, Identifier) and _identifier_has_wildcard( + child + ): + return True + return False + + @staticmethod + def _has_subquery(stmt: sqlparse.sql.Statement) -> bool: + """Recursively walk the AST looking for a SELECT inside parentheses.""" + + def _walk(token: sqlparse.sql.Token) -> bool: + if isinstance(token, Parenthesis): + for child in token.flatten(): + if child.ttype is DML and child.normalized == "SELECT": + return True + if hasattr(token, "tokens"): + for child in token.tokens: + if _walk(child): + return True + return False + + for token in stmt.tokens: + if _walk(token): + return True + return False + + @staticmethod + def _projection_tokens( + stmt: sqlparse.sql.Statement, + ) -> List[sqlparse.sql.Token]: + """Return the non-flattened AST nodes between the first SELECT and FROM.""" + tokens: List[sqlparse.sql.Token] = [] + collecting = False + for token in stmt.tokens: + if token.ttype is DML and token.normalized == "SELECT": + collecting = True + continue + if token.ttype is Keyword and token.normalized in ("FROM", "INTO"): + break + if token.ttype is Keyword and token.normalized == "DISTINCT": + continue + if collecting and token.ttype is not Whitespace: + tokens.append(token) + return tokens + + @staticmethod + def _projection_column_count( + projection: List[sqlparse.sql.Token], + ) -> int: + """Return the number of columns referenced by the projection.""" + for node in projection: + if isinstance(node, IdentifierList): + return len(list(node.get_identifiers())) + if isinstance(node, (Identifier, Function)): + return 1 + if node.ttype is Wildcard: + return 1 + return 0 diff --git a/packages/uipath-platform/src/uipath/platform/entities/_entity_ontology_service.py b/packages/uipath-platform/src/uipath/platform/entities/_entity_ontology_service.py new file mode 100644 index 000000000..a92589f60 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/entities/_entity_ontology_service.py @@ -0,0 +1,80 @@ +"""Ontology-side operations for the Data Fabric entities surface. + +Handles retrieval of ontology component files (OWL schema, R2RML mapping, and +other typed files). Entity schema and record operations are managed by +:class:`EntitySchemaService` / :class:`EntityDataService` and exposed alongside +ontology operations through :class:`EntitiesService`. +""" + +from typing import Any, Dict, Optional + +from pydantic import BaseModel, ConfigDict, Field + +from ..common._base_service import BaseService +from ..common._config import UiPathApiConfig +from ..common._execution_context import UiPathExecutionContext +from ..common._folder_context import header_folder +from ..common._models import Endpoint, RequestSpec +from ..orchestrator._folder_service import FolderService + + +class DataFabricOntologyItem(BaseModel): + """A single Data Fabric ontology reference from agent configuration.""" + + model_config = ConfigDict( + validate_by_name=True, validate_by_alias=True, extra="allow" + ) + + name: str + folder_key: str = Field(alias="folderId") + description: Optional[str] = None + id: Optional[str] = None + + +class EntityOntologyService(BaseService): + """HTTP service for Data Fabric ontology file retrieval. + + Backend target: ``datafabric_/api/ontologies``. + + See Also: + https://docs.uipath.com/data-service/automation-cloud/latest/user-guide/introduction + + !!! warning "Preview Feature" + This service is currently experimental. Behavior and parameters are + subject to change in future versions. + """ + + def __init__( + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + folders_service: Optional[FolderService] = None, + ) -> None: + """Initialise the ontology service.""" + super().__init__(config=config, execution_context=execution_context) + self._folders_service = folders_service + + async def get_file_async( + self, + ontology_name: str, + file_type: str = "owl", + folder_key: Optional[str] = None, + ) -> Dict[str, Any]: + """Internal implementation; see :meth:`EntitiesService.get_ontology_file_async`.""" + spec = self._ontology_file_spec(ontology_name, file_type, folder_key) + response = await self.request_async( + spec.method, spec.endpoint, headers=spec.headers + ) + return response.json() + + @staticmethod + def _ontology_file_spec( + ontology_name: str, file_type: str, folder_key: Optional[str] = None + ) -> RequestSpec: + return RequestSpec( + method="GET", + endpoint=Endpoint( + f"datafabric_/api/ontologies/{ontology_name}/files/{file_type}" + ), + headers=header_folder(folder_key, None), + ) diff --git a/packages/uipath-platform/src/uipath/platform/entities/_entity_resolution.py b/packages/uipath-platform/src/uipath/platform/entities/_entity_resolution.py new file mode 100644 index 000000000..2477b26b9 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/entities/_entity_resolution.py @@ -0,0 +1,526 @@ +from __future__ import annotations + +import abc +import asyncio +import logging +from dataclasses import dataclass +from typing import Awaitable, Callable, Dict, Optional + +from ..common._bindings import ( + EntityResourceOverwrite, + ResourceOverwrite, + _resource_overwrites, +) +from ..common._config import UiPathApiConfig +from ..common._execution_context import UiPathExecutionContext +from ..orchestrator._folder_service import FolderService +from .entities import ( + DataFabricEntityItem, + Entity, + EntityRouting, + QueryRoutingOverrideContext, +) + +FolderPathResolver = Callable[[str], Optional[str]] +AsyncFolderPathResolver = Callable[[str], Awaitable[Optional[str]]] +EntityByKeyFetcher = Callable[[str], Entity] +AsyncEntityByKeyFetcher = Callable[[str], Awaitable[Entity]] +EntityByNameFetcher = Callable[[str, Optional[str]], Entity] +AsyncEntityByNameFetcher = Callable[[str, Optional[str]], Awaitable[Entity]] + + +# --------------------------------------------------------------------------- +# Routing strategy +# --------------------------------------------------------------------------- + + +class RoutingStrategy(abc.ABC): + """Strategy for resolving a ``QueryRoutingOverrideContext`` at query time.""" + + @abc.abstractmethod + def resolve(self) -> Optional[QueryRoutingOverrideContext]: ... + + @abc.abstractmethod + async def resolve_async(self) -> Optional[QueryRoutingOverrideContext]: ... + + +class PreResolvedRoutingStrategy(RoutingStrategy): + """Returns a routing context that was fully resolved at init time. + + Used after ``resolve_entity_set`` where all folder paths have already + been converted to folder keys and the routing context is immutable. + """ + + def __init__( + self, + routing_context: QueryRoutingOverrideContext, + ) -> None: + self._routing_context = routing_context + + def resolve(self) -> Optional[QueryRoutingOverrideContext]: + return self._routing_context + + async def resolve_async(self) -> Optional[QueryRoutingOverrideContext]: + return self._routing_context + + @property + def routing_context(self) -> QueryRoutingOverrideContext: + return self._routing_context + + +class FoldersMapRoutingStrategy(RoutingStrategy): + """Builds a routing context from a pre-populated folders map. + + Used when an ``EntitiesService`` is constructed with an explicit + ``folders_map`` (and optional entity-name overrides) but *without* a + pre-built routing context. Folder paths in the map are resolved to + folder keys lazily at query time via ``FolderService``. + """ + + def __init__( + self, + folders_map: Dict[str, str], + effective_entity_names: Dict[str, str], + folders_service: Optional[FolderService], + ) -> None: + self._folders_map = folders_map + self._effective_entity_names = effective_entity_names + self._folders_service = folders_service + + def resolve(self) -> Optional[QueryRoutingOverrideContext]: + resolved = self._resolve_folder_paths() + return build_resolution_routing_context( + { + name: (resolved or {}).get(path, path) + for name, path in self._folders_map.items() + }, + self._effective_entity_names, + ) + + async def resolve_async(self) -> Optional[QueryRoutingOverrideContext]: + resolved = await self._resolve_folder_paths_async() + return build_resolution_routing_context( + { + name: (resolved or {}).get(path, path) + for name, path in self._folders_map.items() + }, + self._effective_entity_names, + ) + + def _resolve_folder_paths(self) -> Optional[dict[str, str]]: + folder_paths = set(self._folders_map.values()) + if not folder_paths: + return None + + resolved: dict[str, str] = {} + for folder_path in folder_paths: + if self._folders_service is not None: + folder_key = self._folders_service.retrieve_key(folder_path=folder_path) + if folder_key is not None: + resolved[folder_path] = folder_key + continue + resolved[folder_path] = folder_path + return resolved + + async def _resolve_folder_paths_async(self) -> Optional[dict[str, str]]: + folder_paths = set(self._folders_map.values()) + if not folder_paths: + return None + + resolved: dict[str, str] = {} + for folder_path in folder_paths: + if self._folders_service is not None: + folder_key = await self._folders_service.retrieve_key_async( + folder_path=folder_path + ) + if folder_key is not None: + resolved[folder_path] = folder_key + continue + resolved[folder_path] = folder_path + return resolved + + +class ContextOverwriteRoutingStrategy(RoutingStrategy): + """Builds a routing context lazily from ``_resource_overwrites``. + + This is the fallback for direct SDK usage where no ``folders_map`` or + pre-resolved routing context exists. Entity overwrites are read from + the active ``ResourceOverwritesContext`` at query time. + """ + + def __init__(self, folders_service: Optional[FolderService]) -> None: + self._folders_service = folders_service + + def resolve(self) -> Optional[QueryRoutingOverrideContext]: + entity_overwrites = _get_entity_overwrites_from_context() + if not entity_overwrites: + return None + + folder_paths = { + ow.folder_path for ow in entity_overwrites.values() if ow.folder_path + } + resolved = self._resolve_paths(folder_paths) + return self._build(entity_overwrites, resolved) + + async def resolve_async(self) -> Optional[QueryRoutingOverrideContext]: + entity_overwrites = _get_entity_overwrites_from_context() + if not entity_overwrites: + return None + + folder_paths = { + ow.folder_path for ow in entity_overwrites.values() if ow.folder_path + } + resolved = await self._resolve_paths_async(folder_paths) + return self._build(entity_overwrites, resolved) + + def _resolve_paths(self, folder_paths: set[str]) -> dict[str, str]: + resolved: dict[str, str] = {} + for path in folder_paths: + if self._folders_service is not None: + key = self._folders_service.retrieve_key(folder_path=path) + if key is not None: + resolved[path] = key + continue + resolved[path] = path + return resolved + + async def _resolve_paths_async(self, folder_paths: set[str]) -> dict[str, str]: + resolved: dict[str, str] = {} + for path in folder_paths: + if self._folders_service is not None: + key = await self._folders_service.retrieve_key_async(folder_path=path) + if key is not None: + resolved[path] = key + continue + resolved[path] = path + return resolved + + @staticmethod + def _build( + entity_overwrites: Dict[str, EntityResourceOverwrite], + resolved: dict[str, str], + ) -> Optional[QueryRoutingOverrideContext]: + routings: list[EntityRouting] = [] + for original_name, overwrite in entity_overwrites.items(): + override_name = ( + overwrite.resource_identifier + if overwrite.resource_identifier != original_name + else None + ) + folder_id = _resolve_overwrite_folder(overwrite, resolved) + routings.append( + EntityRouting( + entity_name=original_name, + folder_id=folder_id, + override_entity_name=override_name, + ) + ) + + if not routings: + return None + return QueryRoutingOverrideContext(entity_routings=routings) + + +def create_routing_strategy( + *, + folders_map: Optional[Dict[str, str]], + effective_entity_names: Optional[Dict[str, str]], + routing_context: Optional[QueryRoutingOverrideContext], + folders_service: Optional[FolderService], +) -> RoutingStrategy: + """Select the appropriate routing strategy based on init-time state.""" + if routing_context is not None: + return PreResolvedRoutingStrategy(routing_context) + if folders_map: + return FoldersMapRoutingStrategy( + folders_map, + effective_entity_names or {}, + folders_service, + ) + return ContextOverwriteRoutingStrategy(folders_service) + + +# --------------------------------------------------------------------------- +# Helpers shared across strategies +# --------------------------------------------------------------------------- + + +def _get_entity_overwrites_from_context() -> Dict[str, EntityResourceOverwrite]: + """Extract entity overwrites from the active ResourceOverwritesContext.""" + context_overwrites = _resource_overwrites.get() + if not context_overwrites: + return {} + + result: Dict[str, EntityResourceOverwrite] = {} + for key, overwrite in context_overwrites.items(): + if isinstance(overwrite, EntityResourceOverwrite): + original_name = key.split(".", 1)[1] if "." in key else key + result[original_name] = overwrite + return result + + +def _resolve_overwrite_folder( + overwrite: EntityResourceOverwrite, + resolved: dict[str, str], +) -> str: + """Return the folder key for an entity overwrite. + + Uses folder_id directly when present (already a key). + Falls back to resolving folder_path through the resolved map. + """ + if overwrite.folder_id: + return overwrite.folder_id + if overwrite.folder_path and resolved: + return resolved.get(overwrite.folder_path, overwrite.folder_path) + return overwrite.folder_identifier + + +# --------------------------------------------------------------------------- +# Resolution plan (used by resolve_entity_set) +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class EntityFetchByKey: + entity_key: str + + +@dataclass(frozen=True) +class EntityFetchByName: + entity_name: str + folder_key: str + + +@dataclass(frozen=True) +class EntityResolutionDraft: + fetch_by_key: list[EntityFetchByKey] + fetch_by_name: list[EntityFetchByName] + folders_map: dict[str, str] + effective_entity_names: dict[str, str] + folder_paths_to_resolve: set[str] + + +@dataclass(frozen=True) +class EntityResolutionPlan: + fetch_by_key: list[EntityFetchByKey] + fetch_by_name: list[EntityFetchByName] + folders_map: dict[str, str] + effective_entity_names: dict[str, str] + routing_context: QueryRoutingOverrideContext | None + + +def create_resolution_draft( + items: list[DataFabricEntityItem], + context_overwrites: dict[str, ResourceOverwrite], +) -> EntityResolutionDraft: + folders_map: dict[str, str] = {} + effective_entity_names: dict[str, str] = {} + folder_paths_to_resolve: set[str] = set() + fetch_by_key: list[EntityFetchByKey] = [] + fetch_by_name: list[EntityFetchByName] = [] + + for item in items: + overwrite = context_overwrites.get( + f"entity.{item.id}" + ) or context_overwrites.get(f"entity.{item.name}") + resolved_folder = item.folder_key + + if isinstance(overwrite, EntityResourceOverwrite): + folder_changed = False + if overwrite.folder_id: + resolved_folder = overwrite.folder_id + folder_changed = resolved_folder != item.folder_key + elif overwrite.folder_path: + resolved_folder = overwrite.folder_path + folder_changed = True + folder_paths_to_resolve.add(overwrite.folder_path) + + if overwrite.name != item.name or folder_changed: + if overwrite.name != item.name: + effective_entity_names[item.name] = overwrite.name + fetch_by_name.append( + EntityFetchByName( + entity_name=overwrite.name, + folder_key=resolved_folder, + ) + ) + folders_map[item.name] = resolved_folder + continue + + fetch_by_key.append(EntityFetchByKey(entity_key=item.entity_key or item.id)) + folders_map[item.name] = resolved_folder + + return EntityResolutionDraft( + fetch_by_key=fetch_by_key, + fetch_by_name=fetch_by_name, + folders_map=folders_map, + effective_entity_names=effective_entity_names, + folder_paths_to_resolve=folder_paths_to_resolve, + ) + + +def finalize_resolution_plan( + draft: EntityResolutionDraft, + resolve_folder_path: Callable[[str], Optional[str]], +) -> EntityResolutionPlan: + resolved_paths: dict[str, str] = {} + for folder_path in draft.folder_paths_to_resolve: + resolved_paths[folder_path] = resolve_folder_path(folder_path) or folder_path + + resolved_folders_map = { + entity_name: resolved_paths.get(folder_key, folder_key) + for entity_name, folder_key in draft.folders_map.items() + } + resolved_fetch_by_name = [ + EntityFetchByName( + entity_name=entry.entity_name, + folder_key=resolved_paths.get(entry.folder_key, entry.folder_key), + ) + for entry in draft.fetch_by_name + ] + + return EntityResolutionPlan( + fetch_by_key=draft.fetch_by_key, + fetch_by_name=resolved_fetch_by_name, + folders_map=resolved_folders_map, + effective_entity_names=draft.effective_entity_names, + routing_context=build_resolution_routing_context( + resolved_folders_map, + draft.effective_entity_names, + ), + ) + + +def build_resolution_routing_context( + folders_map: dict[str, str], + effective_entity_names: dict[str, str], +) -> QueryRoutingOverrideContext | None: + routings = [ + EntityRouting( + entity_name=original_name, + folder_id=folder_id, + override_entity_name=effective_entity_names.get(original_name), + ) + for original_name, folder_id in folders_map.items() + ] + if not routings: + return None + + return QueryRoutingOverrideContext(entity_routings=routings) + + +def create_resolution_plan( + items: list[DataFabricEntityItem], + context_overwrites: dict[str, ResourceOverwrite], + resolve_folder_path: FolderPathResolver, +) -> EntityResolutionPlan: + draft = create_resolution_draft(items, context_overwrites) + return finalize_resolution_plan(draft, resolve_folder_path) + + +async def create_resolution_plan_async( + items: list[DataFabricEntityItem], + context_overwrites: dict[str, ResourceOverwrite], + resolve_folder_path: AsyncFolderPathResolver, +) -> EntityResolutionPlan: + draft = create_resolution_draft(items, context_overwrites) + folder_paths = list(draft.folder_paths_to_resolve) + results = await asyncio.gather(*(resolve_folder_path(fp) for fp in folder_paths)) + resolved_paths = { + fp: result or fp for fp, result in zip(folder_paths, results, strict=True) + } + + return finalize_resolution_plan( + draft, + lambda folder_path: resolved_paths.get(folder_path, folder_path), + ) + + +def fetch_resolved_entities( + plan: EntityResolutionPlan, + retrieve_by_key: EntityByKeyFetcher, + retrieve_by_name: EntityByNameFetcher, + logger: logging.Logger, +) -> list[Entity]: + entities: list[Entity] = [] + for key_entry in plan.fetch_by_key: + try: + entities.append(retrieve_by_key(key_entry.entity_key)) + except Exception: + logger.warning( + "Failed to fetch entity by key '%s', skipping.", + key_entry.entity_key, + exc_info=True, + ) + + for name_entry in plan.fetch_by_name: + try: + entities.append( + retrieve_by_name(name_entry.entity_name, name_entry.folder_key) + ) + except Exception: + logger.warning( + "Failed to fetch entity by name '%s' (folder_key=%s), skipping.", + name_entry.entity_name, + name_entry.folder_key, + exc_info=True, + ) + + return entities + + +async def fetch_resolved_entities_async( + plan: EntityResolutionPlan, + retrieve_by_key: AsyncEntityByKeyFetcher, + retrieve_by_name: AsyncEntityByNameFetcher, + logger: logging.Logger, +) -> list[Entity]: + async def _safe_fetch_by_key(entry: EntityFetchByKey) -> Optional[Entity]: + try: + return await retrieve_by_key(entry.entity_key) + except Exception: + logger.warning( + "Failed to fetch entity by key '%s', skipping.", + entry.entity_key, + exc_info=True, + ) + return None + + async def _safe_fetch_by_name(entry: EntityFetchByName) -> Optional[Entity]: + try: + return await retrieve_by_name( + entry.entity_name, + entry.folder_key, + ) + except Exception: + logger.warning( + "Failed to fetch entity by name '%s' (folder_key=%s), skipping.", + entry.entity_name, + entry.folder_key, + exc_info=True, + ) + return None + + tasks = [_safe_fetch_by_key(entry) for entry in plan.fetch_by_key] + [ + _safe_fetch_by_name(entry) for entry in plan.fetch_by_name + ] + results = await asyncio.gather(*tasks) + return [entity for entity in results if entity is not None] + + +def build_resolution_service( + *, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + folders_service: FolderService | None, + plan: EntityResolutionPlan, + service_factory: Callable[..., object], +) -> object: + return service_factory( + config=config, + execution_context=execution_context, + folders_service=folders_service, + folders_map=plan.folders_map, + entity_name_overrides=plan.effective_entity_names, + routing_context=plan.routing_context, + ) diff --git a/packages/uipath-platform/src/uipath/platform/entities/_entity_schema_service.py b/packages/uipath-platform/src/uipath/platform/entities/_entity_schema_service.py new file mode 100644 index 000000000..a73b894ec --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/entities/_entity_schema_service.py @@ -0,0 +1,523 @@ +"""Schema-side operations for the Data Fabric entities surface. + +Handles entity definitions, choice set listings, and the create / delete / +update-metadata lifecycle that targets the backend ``EntityController``. +Record CRUD, queries, attachments, and bulk import live on +:class:`EntityDataService` and are mediated by :class:`EntitiesService`. +""" + +import re +from typing import Any, Dict, List, Optional + +from httpx import Response + +from uipath.platform.constants import HEADER_FOLDER_KEY + +from ..common._base_service import BaseService +from ..common._config import UiPathApiConfig +from ..common._execution_context import UiPathExecutionContext +from ..common._models import Endpoint, RequestSpec +from ..orchestrator._folder_service import FolderService +from .entities import ( + ENTITY_FIELD_CONSTRAINT_DEFAULTS, + ENTITY_FIELD_CONSTRAINT_SPEC, + ENTITY_SCHEMA_FIELD_TYPE_MAP, + RESERVED_FIELD_NAMES, + Entity, + EntityCreateFieldOptions, + EntityCreateOptions, + EntityFieldDataType, + EntityMetadataUpdateOptions, +) + +DATA_FABRIC_TENANT_FOLDER_ID = "00000000-0000-0000-0000-000000000000" + +_NAME_RE = re.compile(r"^[a-zA-Z][a-zA-Z0-9]*$") +"""Entity and field name pattern: must start with a letter, then letters and digits only. + +Matches the UI's create-entity / create-field form validators so any name accepted +here can later be displayed or edited through the Data Service UI. +""" + +_ENTITY_NAME_MIN_LENGTH = 1 +_ENTITY_NAME_MAX_LENGTH = 30 +_FIELD_NAME_MIN_LENGTH = 3 +_FIELD_NAME_MAX_LENGTH = 100 + + +class EntitySchemaService(BaseService): + """HTTP service for entity-schema operations. + + Provides retrieval and lifecycle management for entities and choice sets. + Backend target: ``datafabric_/api/Entity``. + + See Also: + https://docs.uipath.com/data-service/automation-cloud/latest/user-guide/introduction + + !!! warning "Preview Feature" + This service is currently experimental. Behavior and parameters are + subject to change in future versions. + """ + + def __init__( + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + folders_service: Optional[FolderService] = None, + ) -> None: + """Initialise the schema service.""" + super().__init__(config=config, execution_context=execution_context) + self._folders_service = folders_service + + def retrieve(self, entity_key: str) -> Entity: + """Internal implementation; see :meth:`EntitiesService.retrieve`.""" + spec = self._retrieve_spec(entity_key) + response = self.request(spec.method, spec.endpoint) + return Entity.model_validate(response.json()) + + async def retrieve_async(self, entity_key: str) -> Entity: + """Async variant of :meth:`retrieve`.""" + spec = self._retrieve_spec(entity_key) + response = await self.request_async(spec.method, spec.endpoint) + return Entity.model_validate(response.json()) + + def retrieve_by_name( + self, entity_name: str, folder_key: Optional[str] = None + ) -> Entity: + """Internal implementation; see :meth:`EntitiesService.retrieve_by_name`.""" + spec = self._retrieve_by_name_spec(entity_name) + headers = self._folder_key_headers(folder_key) + response = self.request(spec.method, spec.endpoint, headers=headers) + return Entity.model_validate(response.json()) + + async def retrieve_by_name_async( + self, entity_name: str, folder_key: Optional[str] = None + ) -> Entity: + """Async variant of :meth:`retrieve_by_name`.""" + spec = self._retrieve_by_name_spec(entity_name) + headers = self._folder_key_headers(folder_key) + response = await self.request_async(spec.method, spec.endpoint, headers=headers) + return Entity.model_validate(response.json()) + + def list_entities(self) -> List[Entity]: + """Internal implementation; see :meth:`EntitiesService.list_entities`.""" + spec = self._list_entities_spec() + response = self.request(spec.method, spec.endpoint) + entities_data = response.json() + return [Entity.model_validate(entity) for entity in entities_data] + + async def list_entities_async(self) -> List[Entity]: + """Async variant of :meth:`list_entities`.""" + spec = self._list_entities_spec() + response = await self.request_async(spec.method, spec.endpoint) + entities_data = response.json() + return [Entity.model_validate(entity) for entity in entities_data] + + def list_choicesets(self) -> List[Entity]: + """Internal implementation; see :meth:`EntitiesService.list_choicesets`.""" + spec = self._list_choicesets_spec() + response = self.request(spec.method, spec.endpoint) + return [Entity.model_validate(item) for item in response.json()] + + async def list_choicesets_async(self) -> List[Entity]: + """Async variant of :meth:`list_choicesets`.""" + spec = self._list_choicesets_spec() + response = await self.request_async(spec.method, spec.endpoint) + return [Entity.model_validate(item) for item in response.json()] + + def create_entity( + self, + name: str, + fields: List[EntityCreateFieldOptions], + options: Optional[EntityCreateOptions] = None, + ) -> str: + """Internal implementation; see :meth:`EntitiesService.create_entity`.""" + spec = self._create_entity_spec(name, fields, options) + response = self.request(spec.method, spec.endpoint, json=spec.json) + return self._extract_entity_id(response) + + async def create_entity_async( + self, + name: str, + fields: List[EntityCreateFieldOptions], + options: Optional[EntityCreateOptions] = None, + ) -> str: + """Async variant of :meth:`create_entity`.""" + spec = self._create_entity_spec(name, fields, options) + response = await self.request_async(spec.method, spec.endpoint, json=spec.json) + return self._extract_entity_id(response) + + def delete_entity(self, entity_id: str) -> None: + """Delete an entity and all of its records.""" + spec = self._delete_entity_spec(entity_id) + self.request(spec.method, spec.endpoint) + + async def delete_entity_async(self, entity_id: str) -> None: + """Async variant of :meth:`delete_entity`.""" + spec = self._delete_entity_spec(entity_id) + await self.request_async(spec.method, spec.endpoint) + + def update_entity_metadata( + self, + entity_id: str, + metadata: EntityMetadataUpdateOptions | Dict[str, Any], + ) -> None: + """Internal implementation; see :meth:`EntitiesService.update_entity_metadata`.""" + spec = self._update_entity_metadata_spec(entity_id, metadata) + self.request(spec.method, spec.endpoint, json=spec.json) + + async def update_entity_metadata_async( + self, + entity_id: str, + metadata: EntityMetadataUpdateOptions | Dict[str, Any], + ) -> None: + """Async variant of :meth:`update_entity_metadata`.""" + spec = self._update_entity_metadata_spec(entity_id, metadata) + await self.request_async(spec.method, spec.endpoint, json=spec.json) + + # ------------------------------------------------------------------ + # Request-spec builders + # ------------------------------------------------------------------ + + @staticmethod + def _retrieve_spec(entity_key: str) -> RequestSpec: + """Build the GET spec for fetching an entity by key.""" + return RequestSpec( + method="GET", + endpoint=Endpoint(f"datafabric_/api/Entity/{entity_key}"), + ) + + @staticmethod + def _retrieve_by_name_spec(entity_name: str) -> RequestSpec: + """Build the GET spec for fetching an entity by name.""" + return RequestSpec( + method="GET", + endpoint=Endpoint(f"datafabric_/api/Entity/{entity_name}/metadata"), + ) + + @staticmethod + def _folder_key_headers(folder_key: Optional[str]) -> Dict[str, str]: + """Return the folder-key header dict, empty when no key is supplied.""" + if folder_key: + return {HEADER_FOLDER_KEY: folder_key} + return {} + + @staticmethod + def _list_entities_spec() -> RequestSpec: + """Build the GET spec for listing all entities (non-choice-sets).""" + return RequestSpec( + method="GET", + endpoint=Endpoint("datafabric_/api/Entity"), + ) + + @staticmethod + def _list_choicesets_spec() -> RequestSpec: + """Build the GET spec for listing all choice sets.""" + return RequestSpec( + method="GET", + endpoint=Endpoint("datafabric_/api/Entity/choiceset"), + ) + + @classmethod + def _create_entity_spec( + cls, + name: str, + fields: List[EntityCreateFieldOptions], + options: Optional[EntityCreateOptions] = None, + ) -> RequestSpec: + """Build the POST spec for creating an entity with its field schema.""" + cls._validate_name(name, "entity") + for field in fields: + cls._validate_name(field.field_name, "field") + opts = options or EntityCreateOptions() + # The user-facing option ``is_analytics_enabled`` maps to the legacy + # backend field name ``isInsightsEnabled`` — the wire name predates + # the "Analytics" UI rename. + payload: Dict[str, Any] = { + "displayName": opts.display_name or name, + "entityDefinition": { + "name": name, + "fields": [cls._build_schema_field_payload(f) for f in fields], + "folderId": opts.folder_key or DATA_FABRIC_TENANT_FOLDER_ID, + "isRbacEnabled": bool(opts.is_rbac_enabled or False), + "isInsightsEnabled": bool(opts.is_analytics_enabled or False), + "externalFields": opts.external_fields or [], + }, + } + if opts.description is not None: + payload["description"] = opts.description + return RequestSpec( + method="POST", + endpoint=Endpoint("datafabric_/api/Entity"), + json=payload, + ) + + @staticmethod + def _delete_entity_spec(entity_id: str) -> RequestSpec: + """Build the DELETE spec for removing an entity.""" + return RequestSpec( + method="DELETE", + endpoint=Endpoint(f"datafabric_/api/Entity/{entity_id}"), + ) + + @staticmethod + def _update_entity_metadata_spec( + entity_id: str, + metadata: EntityMetadataUpdateOptions | Dict[str, Any], + ) -> RequestSpec: + """Build the PATCH spec for updating entity metadata. + + Dict inputs are validated through :class:`EntityMetadataUpdateOptions` + so snake_case keys (``display_name``) and camelCase keys + (``displayName``) both serialise to the API field names the backend + expects. + """ + if not isinstance(metadata, EntityMetadataUpdateOptions): + metadata = EntityMetadataUpdateOptions.model_validate(metadata) + body = metadata.model_dump(by_alias=True, exclude_none=True) + return RequestSpec( + method="PATCH", + endpoint=Endpoint(f"datafabric_/api/Entity/{entity_id}/metadata"), + json=body, + ) + + @classmethod + def _build_schema_field_payload( + cls, field: EntityCreateFieldOptions + ) -> Dict[str, Any]: + """Build the API field payload for a single field on create-entity. + + Maps :class:`EntityFieldDataType` to the backend's ``sqlType.name`` and + ``fieldDisplayType`` (e.g. ``STRING`` becomes ``NVARCHAR`` / ``Basic``). + Caller-supplied constraints are validated against + :data:`ENTITY_FIELD_CONSTRAINT_SPEC`; unsupplied per-type constraints + fall back to :data:`ENTITY_FIELD_CONSTRAINT_DEFAULTS` so the field is + persisted fully and remains editable later. + """ + ftype = field.type or EntityFieldDataType.STRING + cls._validate_name(field.field_name, "field") + cls._validate_field_constraints(ftype, field) + + sql_type_name, field_display_type = ENTITY_SCHEMA_FIELD_TYPE_MAP[ftype] + sql_type: Dict[str, Any] = {"name": sql_type_name} + sql_type.update(cls._build_sql_type_constraints(ftype, field)) + + payload: Dict[str, Any] = { + "name": field.field_name, + "displayName": field.display_name or field.field_name, + "sqlType": sql_type, + "fieldDisplayType": field_display_type, + "description": field.description or "", + "isRequired": bool(field.is_required or False), + "isUnique": bool(field.is_unique or False), + "isRbacEnabled": bool(field.is_rbac_enabled or False), + "isEncrypted": bool(field.is_encrypted or False), + } + if field.default_value is not None: + payload["defaultValue"] = field.default_value + if field.choice_set_id is not None: + payload["choiceSetId"] = field.choice_set_id + if field.reference_entity_name is not None: + payload["referenceEntityName"] = field.reference_entity_name + if field.reference_field_name is not None: + payload["referenceFieldName"] = field.reference_field_name + return payload + + @staticmethod + def _build_sql_type_constraints( + ftype: EntityFieldDataType, field: EntityCreateFieldOptions + ) -> Dict[str, Any]: + """Return the ``sqlType`` constraint fields required for ``ftype``. + + Caller-supplied values override defaults where the type accepts them; + types that take no constraints (UUID, DATETIME, CHOICE_SET_SINGLE, + AUTO_NUMBER) return an empty dict. + """ + d = ENTITY_FIELD_CONSTRAINT_DEFAULTS + if ftype is EntityFieldDataType.STRING: + return {"lengthLimit": field.length_limit or d["STRING_LENGTH_LIMIT"]} + if ftype is EntityFieldDataType.MULTILINE_TEXT: + return { + "lengthLimit": field.length_limit or d["MULTILINE_TEXT_LENGTH_LIMIT"] + } + if ftype is EntityFieldDataType.DECIMAL: + return { + "lengthLimit": d["DECIMAL_LENGTH_LIMIT"], + "decimalPrecision": ( + field.decimal_precision + if field.decimal_precision is not None + else d["DECIMAL_PRECISION"] + ), + "maxValue": ( + field.max_value + if field.max_value is not None + else d["NUMERIC_MAX_VALUE"] + ), + "minValue": ( + field.min_value + if field.min_value is not None + else d["NUMERIC_MIN_VALUE"] + ), + } + if ftype is EntityFieldDataType.BOOLEAN: + return {"lengthLimit": d["BOOLEAN_LENGTH_LIMIT"]} + if ftype in ( + EntityFieldDataType.DATE, + EntityFieldDataType.DATETIME_WITH_TZ, + ): + return {"lengthLimit": d["DATE_LENGTH_LIMIT"]} + if ftype in (EntityFieldDataType.INTEGER, EntityFieldDataType.BIG_INTEGER): + return { + "maxValue": ( + field.max_value + if field.max_value is not None + else d["NUMERIC_MAX_VALUE"] + ), + "minValue": ( + field.min_value + if field.min_value is not None + else d["NUMERIC_MIN_VALUE"] + ), + } + if ftype in (EntityFieldDataType.FLOAT, EntityFieldDataType.DOUBLE): + return { + "decimalPrecision": ( + field.decimal_precision + if field.decimal_precision is not None + else d["DECIMAL_PRECISION"] + ), + "maxValue": ( + field.max_value + if field.max_value is not None + else d["NUMERIC_MAX_VALUE"] + ), + "minValue": ( + field.min_value + if field.min_value is not None + else d["NUMERIC_MIN_VALUE"] + ), + } + if ftype in (EntityFieldDataType.FILE, EntityFieldDataType.RELATIONSHIP): + return {"lengthLimit": d["UNIQUEIDENTIFIER_LENGTH_LIMIT"]} + if ftype is EntityFieldDataType.CHOICE_SET_MULTIPLE: + return {"lengthLimit": d["CHOICE_SET_MULTIPLE_LENGTH_LIMIT"]} + # UUID, DATETIME, CHOICE_SET_SINGLE, AUTO_NUMBER — no constraints + return {} + + @staticmethod + def _validate_name(name: str, context: str) -> None: + r"""Validate an entity or field name against the UI's create-form rules. + + Entity names must be 1-30 characters; field names must be 3-100 + characters. Both must match ``^[a-zA-Z][a-zA-Z0-9]*$`` — start with a + letter, then letters or digits only (underscores are not permitted, to + stay consistent with the UI's entity / field creation forms). + + Field names additionally cannot collide with the system-reserved field + names in :data:`RESERVED_FIELD_NAMES`; the reserved-name check runs + first so that short reserved names produce a more informative error. + """ + if context == "field": + if name in RESERVED_FIELD_NAMES: + reserved = ", ".join(sorted(RESERVED_FIELD_NAMES)) + raise ValueError( + f"Field name {name!r} is reserved. Reserved names: {reserved}." + ) + min_len, max_len = _FIELD_NAME_MIN_LENGTH, _FIELD_NAME_MAX_LENGTH + else: + min_len, max_len = _ENTITY_NAME_MIN_LENGTH, _ENTITY_NAME_MAX_LENGTH + + if not (min_len <= len(name) <= max_len) or not _NAME_RE.match(name): + raise ValueError( + f"Invalid {context} name {name!r}. Must start with a letter, " + f"contain only letters and digits, and be {min_len}-{max_len} " + "characters." + ) + + @staticmethod + def _validate_field_constraints( + ftype: EntityFieldDataType, field: EntityCreateFieldOptions + ) -> None: + """Validate caller-supplied per-field constraints. + + Rejects constraints that ``ftype`` does not accept (e.g. + ``decimal_precision`` on ``STRING``), values outside the inclusive + range declared in :data:`ENTITY_FIELD_CONSTRAINT_SPEC`, and + ``min_value`` greater than or equal to ``max_value`` when both are + supplied. Also enforces type-dependent required references: + ``CHOICE_SET_SINGLE`` and ``CHOICE_SET_MULTIPLE`` need + ``choice_set_id``; ``RELATIONSHIP`` needs ``reference_entity_name``. + """ + if ( + ftype + in ( + EntityFieldDataType.CHOICE_SET_SINGLE, + EntityFieldDataType.CHOICE_SET_MULTIPLE, + ) + and not field.choice_set_id + ): + raise ValueError( + f"Field {field.field_name!r} of type {ftype.value} requires " + "choice_set_id." + ) + if ( + ftype is EntityFieldDataType.RELATIONSHIP + and not field.reference_entity_name + ): + raise ValueError( + f"Field {field.field_name!r} of type {ftype.value} requires " + "reference_entity_name." + ) + + spec = ENTITY_FIELD_CONSTRAINT_SPEC.get(ftype, {}) + provided: Dict[str, Any] = {} + for attr in ("length_limit", "max_value", "min_value", "decimal_precision"): + value = getattr(field, attr) + if value is not None: + provided[attr] = value + + unsupported = [name for name in provided if name not in spec] + if unsupported: + allowed = ", ".join(sorted(spec.keys())) or "none" + raise ValueError( + f"Field {field.field_name!r} of type {ftype.value} does not accept " + f"{', '.join(sorted(unsupported))}. Allowed constraints: {allowed}." + ) + + for name, value in provided.items(): + low, high = spec[name] + if not (low <= value <= high): + raise ValueError( + f"Field {field.field_name!r} of type {ftype.value}: " + f"{name}={value} is out of range [{low}, {high}]." + ) + + if ( + field.min_value is not None + and field.max_value is not None + and field.min_value >= field.max_value + ): + raise ValueError( + f"Field {field.field_name!r}: min_value ({field.min_value}) must be " + f"strictly less than max_value ({field.max_value})." + ) + + @staticmethod + def _extract_entity_id(response: Response) -> str: + """Return the new entity id from a create-entity response. + + Accepts both a bare JSON string id and a JSON object containing + ``id`` or ``entityId``. + """ + try: + body = response.json() + except Exception: + return response.text.strip().strip('"') + if isinstance(body, str): + return body + if isinstance(body, dict): + for key in ("id", "Id", "entityId", "EntityId"): + value = body.get(key) + if isinstance(value, str): + return value + return response.text.strip().strip('"') diff --git a/packages/uipath-platform/src/uipath/platform/entities/entities.py b/packages/uipath-platform/src/uipath/platform/entities/entities.py index b2c49b763..51eea2d1b 100644 --- a/packages/uipath-platform/src/uipath/platform/entities/entities.py +++ b/packages/uipath-platform/src/uipath/platform/entities/entities.py @@ -1,10 +1,34 @@ """Entities models for UiPath Platform API interactions.""" -from enum import Enum -from types import EllipsisType -from typing import Any, Dict, List, Optional, Type, Union, get_args, get_origin +from __future__ import annotations -from pydantic import BaseModel, ConfigDict, Field, create_model +from enum import Enum, IntEnum +from types import EllipsisType +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterator, + List, + Optional, + Type, + Union, + get_args, + get_origin, + overload, +) + +from pydantic import ( + AliasChoices, + BaseModel, + ConfigDict, + Field, + create_model, + model_validator, +) + +if TYPE_CHECKING: + from ._entities_service import EntitiesService class ReferenceType(Enum): @@ -67,8 +91,8 @@ class ExternalConnection(BaseModel): id: str connection_id: str = Field(alias="connectionId") element_instance_id: str = Field(alias="elementInstanceId") - folder_id: str = Field(alias="folderKey") # named folderKey in TS SDK - connector_id: str = Field(alias="connectorKey") # named connectorKey in TS SDK + folder_id: str = Field(alias="folderKey") + connector_id: str = Field(alias="connectorKey") connector_name: str = Field(alias="connectorName") connection_name: str = Field(alias="connectionName") @@ -125,7 +149,7 @@ class FieldMetadata(BaseModel): reference_field: Optional["EntityField"] = Field( default=None, alias="referenceField" ) - reference_type: ReferenceType = Field(alias="referenceType") + reference_type: Optional[ReferenceType] = Field(default=None, alias="referenceType") sql_type: "FieldDataType" = Field(alias="sqlType") is_required: bool = Field(alias="isRequired") display_name: str = Field(alias="displayName") @@ -197,14 +221,40 @@ class SourceJoinCriteria(BaseModel): model_config = ConfigDict( validate_by_name=True, validate_by_alias=True, + extra="allow", + ) + id: Optional[str] = None + entity_id: Optional[str] = Field(default=None, alias="entityId") + join_field_name: Optional[str] = Field(default=None, alias="joinFieldName") + join_type: Optional[str] = Field(default=None, alias="joinType") + related_source_object_id: Optional[str] = Field( + default=None, alias="relatedSourceObjectId" + ) + related_source_object_field_name: Optional[str] = Field( + default=None, alias="relatedSourceObjectFieldName" + ) + related_source_field_name: Optional[str] = Field( + default=None, alias="relatedSourceFieldName" ) - id: str - entity_id: str = Field(alias="entityId") - join_field_name: str = Field(alias="joinFieldName") - join_type: str = Field(alias="joinType") - related_source_object_id: str = Field(alias="relatedSourceObjectId") - related_source_object_field_name: str = Field(alias="relatedSourceObjectFieldName") - related_source_field_name: str = Field(alias="relatedSourceFieldName") + + +class ChoiceSetValue(BaseModel): + """Model representing a single value within a choice set.""" + + model_config = ConfigDict( + validate_by_name=True, + validate_by_alias=True, + ) + + id: str = Field(alias="Id") + name: str = Field(alias="Name") + display_name: str = Field(alias="DisplayName") + number_id: int = Field(alias="NumberId") + created_time: str | None = Field(default=None, alias="CreateTime") + updated_time: str | None = Field(default=None, alias="UpdateTime") + created_by: str | None = Field(default=None, alias="CreatedBy") + updated_by: str | None = Field(default=None, alias="UpdatedBy") + record_owner: str | None = Field(default=None, alias="RecordOwner") class EntityRecord(BaseModel): @@ -216,7 +266,7 @@ class EntityRecord(BaseModel): "extra": "allow", } - id: str = Field(alias="Id") # Mandatory field validated by Pydantic + id: str = Field(alias="Id") @classmethod def from_data( @@ -292,11 +342,16 @@ class Entity(BaseModel): entity_type: str = Field(alias="entityType") description: Optional[str] = Field(default=None, alias="description") fields: Optional[List[FieldMetadata]] = Field(default=None, alias="fields") - external_fields: Optional[List[ExternalSourceFields]] = Field( - default=None, alias="externalFields" + external_fields: Optional[ + List[ExternalField | ExternalSourceFields | Dict[str, Any]] + ] = Field( + default=None, + alias="externalFields", ) - source_join_criteria: Optional[List[SourceJoinCriteria]] = Field( - default=None, alias="sourceJoinCriteria" + source_join_criteria: Optional[List[SourceJoinCriteria | Dict[str, Any]]] = Field( + default=None, + validation_alias=AliasChoices("sourceJoinCriteria", "sourceJoinCriterias"), + alias="sourceJoinCriteria", ) record_count: Optional[int] = Field(default=None, alias="recordCount") storage_size_in_mb: Optional[float] = Field(default=None, alias="storageSizeInMB") @@ -310,6 +365,25 @@ class Entity(BaseModel): id: str +class FailureRecord(BaseModel): + """A record that failed to insert/update/delete in a batch operation. + + Backend error responses for failed records do not always include a valid + ``Id`` field — this model accepts arbitrary shapes so the caller can + inspect ``error`` text and the original ``record`` payload. + """ + + model_config = ConfigDict( + validate_by_name=True, + validate_by_alias=True, + extra="allow", + ) + + id: Optional[str] = Field(default=None, alias="Id") + error: Optional[str] = Field(default=None) + record: Optional[Dict[str, Any]] = Field(default=None) + + class EntityRecordsBatchResponse(BaseModel): """Model representing a batch response of entity records.""" @@ -318,8 +392,421 @@ class EntityRecordsBatchResponse(BaseModel): validate_by_alias=True, ) - success_records: List[EntityRecord] = Field(alias="successRecords") - failure_records: List[EntityRecord] = Field(alias="failureRecords") + success_records: List[EntityRecord] = Field( + default_factory=list, alias="successRecords" + ) + failure_records: List[FailureRecord] = Field( + default_factory=list, alias="failureRecords" + ) + + +class EntityRecordsListResponse(List[EntityRecord]): + """List of EntityRecord with pagination metadata. + + Subclasses ``list`` so existing call sites that iterate, index, or call + ``len()`` continue to work; new fields ``total_count``, ``has_next_page``, + and ``next_cursor`` expose pagination information returned by the backend. + """ + + def __init__( + self, + items: Optional[List[EntityRecord]] = None, + total_count: int = 0, + has_next_page: bool = False, + next_cursor: Optional[str] = None, + ) -> None: + """Construct from a list of records plus pagination metadata.""" + super().__init__(items or []) + self.total_count = total_count + self.has_next_page = has_next_page + self.next_cursor = next_cursor + + +class LogicalOperator(IntEnum): + """Logical operator for combining query filter groups.""" + + And = 0 + Or = 1 + + +class QueryFilterOperator(str, Enum): + """Comparison operators supported by the structured query API.""" + + Equals = "=" + NotEquals = "!=" + GreaterThan = ">" + LessThan = "<" + GreaterThanOrEqual = ">=" + LessThanOrEqual = "<=" + Contains = "contains" + NotContains = "not contains" + StartsWith = "startswith" + EndsWith = "endswith" + In = "in" + NotIn = "not in" + + +class EntityQueryFilter(BaseModel): + """A single filter condition for querying entity records. + + Backend operator/operand rules: + + * ``in`` / ``not in`` — require a non-empty ``value_list`` and reject + ``value``. + * ``=`` / ``!=`` — allow a null ``value`` (becomes ``IS NULL`` / ``IS + NOT NULL``) and reject ``value_list``. + * All other operators (``>``, ``<``, ``>=``, ``<=``, ``contains``, + ``not contains``, ``startswith``, ``endswith``) — require a non-null + ``value`` and reject ``value_list``. + """ + + model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + + field_name: str = Field(alias="fieldName") + operator: QueryFilterOperator + value: Optional[str] = None + value_list: Optional[List[str]] = Field(default=None, alias="valueList") + + @model_validator(mode="after") + def _check_operator_operands(self) -> "EntityQueryFilter": + """Reject operator/operand combinations the backend rejects. + + Implements the same rules the Data Service ``SelectQueryBuilder`` + enforces server-side, so callers see a clear local error instead of + an opaque HTTP 400. + """ + op = self.operator + if op in (QueryFilterOperator.In, QueryFilterOperator.NotIn): + if not self.value_list: + raise ValueError( + f"Operator {op.value!r} requires a non-empty value_list." + ) + if self.value is not None: + raise ValueError( + f"Operator {op.value!r} uses value_list; value must be omitted." + ) + return self + + if self.value_list is not None: + raise ValueError( + f"Operator {op.value!r} uses value; value_list must be omitted." + ) + if ( + op not in (QueryFilterOperator.Equals, QueryFilterOperator.NotEquals) + and self.value is None + ): + raise ValueError(f"Operator {op.value!r} requires a non-null value.") + return self + + +class EntityQueryFilterGroup(BaseModel): + """A group of query filters combined with a logical operator.""" + + model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + + logical_operator: Optional[LogicalOperator] = Field( + default=None, alias="logicalOperator" + ) + continue_logical_operator: Optional[LogicalOperator] = Field( + default=None, alias="continueLogicalOperator" + ) + query_filters: Optional[List[EntityQueryFilter]] = Field( + default=None, alias="queryFilters" + ) + filter_groups: Optional[List["EntityQueryFilterGroup"]] = Field( + default=None, alias="filterGroups" + ) + + +class EntityQuerySortOption(BaseModel): + """Sort option for query results.""" + + model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + + field_name: str = Field(alias="fieldName") + is_descending: Optional[bool] = Field(default=None, alias="isDescending") + + +class EntityAggregateFunction(str, Enum): + """Aggregate functions supported by the Data Fabric query API.""" + + Count = "COUNT" + Sum = "SUM" + Avg = "AVG" + Min = "MIN" + Max = "MAX" + + +class EntityAggregate(BaseModel): + """A single aggregate expression to apply during a query.""" + + model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + + function: EntityAggregateFunction + field: str + alias: Optional[str] = None + + +class EntityJoin(BaseModel): + """Multi-entity JOIN definition for cross-entity queries.""" + + model_config = ConfigDict( + validate_by_name=True, validate_by_alias=True, extra="allow" + ) + + entity_name: Optional[str] = Field(default=None, alias="entityName") + join_type: Optional[str] = Field(default=None, alias="joinType") + join_field_name: Optional[str] = Field(default=None, alias="joinFieldName") + related_entity_name: Optional[str] = Field(default=None, alias="relatedEntityName") + related_field_name: Optional[str] = Field(default=None, alias="relatedFieldName") + + +class EntityBinning(BaseModel): + """A binning (GROUP BY/aggregation) clause for V2 query endpoint.""" + + model_config = ConfigDict( + validate_by_name=True, validate_by_alias=True, extra="allow" + ) + + field_name: Optional[str] = Field(default=None, alias="fieldName") + aggregate_function: Optional[EntityAggregateFunction] = Field( + default=None, alias="aggregateFunction" + ) + alias: Optional[str] = None + + +class AggregateRow(BaseModel): + """A row returned by aggregate / group-by / binning queries. + + Aggregate rows do not have an ``Id`` field; columns vary by query + (``selected_fields``, ``aggregates`` aliases, binning aliases) and are + accessible as attributes via ``extra="allow"``. + """ + + model_config = ConfigDict( + validate_by_name=True, validate_by_alias=True, extra="allow" + ) + + +class RetrieveEntityRecordsResponse(BaseModel): + """Response from :meth:`EntitiesService.retrieve_records`. + + For plain queries, ``items`` is a list of :class:`EntityRecord`. When the + query uses ``aggregates``, ``group_by``, or ``binnings``, the backend + returns rows without an ``Id`` field; those rows are parsed as + :class:`AggregateRow` instances. + """ + + model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + + items: List[EntityRecord | AggregateRow] = Field(default_factory=list) + total_count: int = Field(default=0, alias="totalCount") + has_next_page: bool = Field(default=False, alias="hasNextPage") + next_cursor: Optional[str] = Field(default=None, alias="nextCursor") + + def __iter__(self) -> Iterator[EntityRecord | AggregateRow]: # type: ignore[override] + """Iterate over records (delegates to ``self.items``).""" + return iter(self.items) + + def __len__(self) -> int: + """Return the number of records (delegates to ``self.items``).""" + return len(self.items) + + @overload + def __getitem__(self, index: int) -> EntityRecord | AggregateRow: ... + + @overload + def __getitem__(self, index: slice) -> List[EntityRecord | AggregateRow]: ... + + def __getitem__( + self, index: int | slice + ) -> EntityRecord | AggregateRow | List[EntityRecord | AggregateRow]: + """Index or slice records (delegates to ``self.items``).""" + return self.items[index] + + +class EntityFieldDataType(str, Enum): + """User-facing entity field data type names accepted by ``create_entity``.""" + + UUID = "UUID" + STRING = "STRING" + INTEGER = "INTEGER" + DATETIME = "DATETIME" + DATETIME_WITH_TZ = "DATETIME_WITH_TZ" + DECIMAL = "DECIMAL" + FLOAT = "FLOAT" + DOUBLE = "DOUBLE" + DATE = "DATE" + BOOLEAN = "BOOLEAN" + BIG_INTEGER = "BIG_INTEGER" + MULTILINE_TEXT = "MULTILINE_TEXT" + FILE = "FILE" + CHOICE_SET_SINGLE = "CHOICE_SET_SINGLE" + CHOICE_SET_MULTIPLE = "CHOICE_SET_MULTIPLE" + AUTO_NUMBER = "AUTO_NUMBER" + RELATIONSHIP = "RELATIONSHIP" + + +# Maps the user-facing EntityFieldDataType to the ``(sqlType.name, fieldDisplayType)`` +# tuple expected by the backend when creating an entity. ``sqlType.name`` is +# the raw SQL Server type the backend persists; ``fieldDisplayType`` controls +# how the field renders in the UI. +ENTITY_SCHEMA_FIELD_TYPE_MAP: Dict[EntityFieldDataType, "tuple[str, str]"] = { + EntityFieldDataType.UUID: ("UNIQUEIDENTIFIER", "Basic"), + EntityFieldDataType.STRING: ("NVARCHAR", "Basic"), + EntityFieldDataType.INTEGER: ("INT", "Basic"), + EntityFieldDataType.DATETIME: ("DATETIME2", "Basic"), + EntityFieldDataType.DATETIME_WITH_TZ: ("DATETIMEOFFSET", "Basic"), + EntityFieldDataType.DECIMAL: ("DECIMAL", "Basic"), + EntityFieldDataType.FLOAT: ("FLOAT", "Basic"), + EntityFieldDataType.DOUBLE: ("REAL", "Basic"), + EntityFieldDataType.DATE: ("DATE", "Basic"), + EntityFieldDataType.BOOLEAN: ("BIT", "Basic"), + EntityFieldDataType.BIG_INTEGER: ("BIGINT", "Basic"), + EntityFieldDataType.MULTILINE_TEXT: ("MULTILINE", "Basic"), + EntityFieldDataType.FILE: ("UNIQUEIDENTIFIER", "File"), + EntityFieldDataType.CHOICE_SET_SINGLE: ("INT", "ChoiceSetSingle"), + EntityFieldDataType.CHOICE_SET_MULTIPLE: ("NVARCHAR", "ChoiceSetMultiple"), + EntityFieldDataType.AUTO_NUMBER: ("DECIMAL", "AutoNumber"), + EntityFieldDataType.RELATIONSHIP: ("UNIQUEIDENTIFIER", "Relationship"), +} + +# Default and fixed sqlType constraint values applied when the caller does +# not supply them. The backend requires these on field creation — without +# them the field is stored in an incomplete state and the UI later fails +# with "Field type cannot be changed" when editing advanced options. +ENTITY_FIELD_CONSTRAINT_DEFAULTS: Dict[str, int] = { + "STRING_LENGTH_LIMIT": 200, + "MULTILINE_TEXT_LENGTH_LIMIT": 200, + "DECIMAL_LENGTH_LIMIT": 1000, + "DECIMAL_PRECISION": 2, + "BOOLEAN_LENGTH_LIMIT": 100, + "DATE_LENGTH_LIMIT": 1000, + "UNIQUEIDENTIFIER_LENGTH_LIMIT": 300, + "CHOICE_SET_MULTIPLE_LENGTH_LIMIT": 4000, + "NUMERIC_MAX_VALUE": 1_000_000_000_000, + "NUMERIC_MIN_VALUE": -1_000_000_000_000, +} + +# Per-field-type spec describing which user-supplied constraints are valid +# and their inclusive ranges. Field types absent from this map (BOOLEAN, +# DATE, DATETIME, DATETIME_WITH_TZ, FILE, RELATIONSHIP, UUID, CHOICE_SET_*, +# AUTO_NUMBER) accept no user-supplied constraints — passing one raises +# ``ValueError`` so the caller gets a clear local error before any HTTP call. +_MAX_SAFE_INTEGER = 9_007_199_254_740_991 + +ENTITY_FIELD_CONSTRAINT_SPEC: Dict[ + EntityFieldDataType, Dict[str, "tuple[int, int]"] +] = { + EntityFieldDataType.STRING: { + "length_limit": (1, 4000), + }, + EntityFieldDataType.MULTILINE_TEXT: { + "length_limit": (1, 10000), + }, + EntityFieldDataType.INTEGER: { + "max_value": (-_MAX_SAFE_INTEGER, _MAX_SAFE_INTEGER), + "min_value": (-_MAX_SAFE_INTEGER, _MAX_SAFE_INTEGER), + }, + EntityFieldDataType.BIG_INTEGER: { + "max_value": (-_MAX_SAFE_INTEGER, _MAX_SAFE_INTEGER), + "min_value": (-_MAX_SAFE_INTEGER, _MAX_SAFE_INTEGER), + }, + EntityFieldDataType.DECIMAL: { + "max_value": (-_MAX_SAFE_INTEGER, _MAX_SAFE_INTEGER), + "min_value": (-_MAX_SAFE_INTEGER, _MAX_SAFE_INTEGER), + "decimal_precision": (0, 10), + }, + EntityFieldDataType.FLOAT: { + "max_value": (-_MAX_SAFE_INTEGER, _MAX_SAFE_INTEGER), + "min_value": (-_MAX_SAFE_INTEGER, _MAX_SAFE_INTEGER), + "decimal_precision": (0, 10), + }, + EntityFieldDataType.DOUBLE: { + "max_value": (-_MAX_SAFE_INTEGER, _MAX_SAFE_INTEGER), + "min_value": (-_MAX_SAFE_INTEGER, _MAX_SAFE_INTEGER), + "decimal_precision": (0, 10), + }, +} + +RESERVED_FIELD_NAMES = frozenset( + ["Id", "CreatedBy", "CreateTime", "UpdatedBy", "UpdateTime"] +) +"""Field names reserved by the backend — using one as a user field name is rejected.""" + + +class EntityCreateFieldOptions(BaseModel): + """User-facing field definition for creating or updating entity schemas.""" + + model_config = ConfigDict( + validate_by_name=True, validate_by_alias=True, extra="allow" + ) + + field_name: str = Field(alias="fieldName") + type: Optional[EntityFieldDataType] = Field( + default=EntityFieldDataType.STRING, alias="type" + ) + display_name: Optional[str] = Field(default=None, alias="displayName") + description: Optional[str] = None + is_required: Optional[bool] = Field(default=None, alias="isRequired") + is_unique: Optional[bool] = Field(default=None, alias="isUnique") + is_rbac_enabled: Optional[bool] = Field(default=None, alias="isRbacEnabled") + is_encrypted: Optional[bool] = Field(default=None, alias="isEncrypted") + default_value: Optional[str] = Field(default=None, alias="defaultValue") + length_limit: Optional[int] = Field(default=None, alias="lengthLimit") + max_value: Optional[int] = Field(default=None, alias="maxValue") + min_value: Optional[int] = Field(default=None, alias="minValue") + decimal_precision: Optional[int] = Field(default=None, alias="decimalPrecision") + choice_set_id: Optional[str] = Field(default=None, alias="choiceSetId") + reference_entity_name: Optional[str] = Field( + default=None, alias="referenceEntityName" + ) + reference_field_name: Optional[str] = Field( + default=None, alias="referenceFieldName" + ) + + +class EntityCreateOptions(BaseModel): + """Options for creating a new Data Fabric entity.""" + + model_config = ConfigDict( + validate_by_name=True, validate_by_alias=True, extra="allow" + ) + + display_name: Optional[str] = Field(default=None, alias="displayName") + description: Optional[str] = None + folder_key: Optional[str] = Field(default=None, alias="folderKey") + is_rbac_enabled: Optional[bool] = Field(default=None, alias="isRbacEnabled") + is_analytics_enabled: Optional[bool] = Field( + default=None, alias="isAnalyticsEnabled" + ) + external_fields: Optional[List[Dict[str, Any]]] = Field( + default=None, alias="externalFields" + ) + + +class EntityMetadataUpdateOptions(BaseModel): + """Options for updating an entity's metadata via PATCH /metadata.""" + + model_config = ConfigDict( + validate_by_name=True, validate_by_alias=True, extra="allow" + ) + + display_name: Optional[str] = Field(default=None, alias="displayName") + description: Optional[str] = None + is_rbac_enabled: Optional[bool] = Field(default=None, alias="isRbacEnabled") + + +class EntityImportRecordsResponse(BaseModel): + """Response from a bulk import operation.""" + + model_config = ConfigDict( + validate_by_name=True, validate_by_alias=True, extra="allow" + ) + + total_records: int = Field(default=0, alias="totalRecords") + inserted_records: int = Field(default=0, alias="insertedRecords") + error_file_link: Optional[str] = Field(default=None, alias="errorFileLink") class EntityRouting(BaseModel): @@ -342,4 +829,28 @@ class QueryRoutingOverrideContext(BaseModel): entity_routings: List[EntityRouting] = Field(alias="entityRoutings") +class DataFabricEntityItem(BaseModel): + """A single Data Fabric entity reference from agent configuration.""" + + model_config = ConfigDict( + validate_by_name=True, validate_by_alias=True, extra="allow" + ) + + id: str + entity_key: Optional[str] = Field(None, alias="referenceKey") + name: str + folder_key: str = Field(alias="folderId") + description: Optional[str] = None + + +class EntitySetResolution(BaseModel): + """Result of resolving an agent entity set with overwrites applied.""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + entities: list[Entity] + entities_service: EntitiesService + + Entity.model_rebuild() +EntityQueryFilterGroup.model_rebuild() diff --git a/packages/uipath-platform/src/uipath/platform/errors/__init__.py b/packages/uipath-platform/src/uipath/platform/errors/__init__.py index 3c446fe14..5ac51d326 100644 --- a/packages/uipath-platform/src/uipath/platform/errors/__init__.py +++ b/packages/uipath-platform/src/uipath/platform/errors/__init__.py @@ -8,6 +8,8 @@ - FolderNotFoundException: Raised when a folder cannot be found - UnsupportedDataSourceException: Raised when an operation is attempted on an unsupported data source type - IngestionInProgressException: Raised when a search is attempted on an index during ingestion +- ContextGroundingIndexNotFoundError: Raised when a context grounding index cannot be resolved by name +- BatchTransformFailedException: Raised when a batch transform has failed - BatchTransformNotCompleteException: Raised when attempting to get results from an incomplete batch transform - OperationNotCompleteException: Raised when attempting to get results from an incomplete operation - OperationFailedException: Raised when an operation has failed @@ -15,7 +17,12 @@ """ from ._base_url_missing_error import BaseUrlMissingError +from ._batch_transform_failed_exception import BatchTransformFailedException from ._batch_transform_not_complete_exception import BatchTransformNotCompleteException +from ._context_grounding_index_not_found_exception import ( + ContextGroundingIndexNotFoundError, +) +from ._datafabric_error import DataFabricError from ._enriched_exception import EnrichedException, ExtractedErrorInfo from ._folder_not_found_exception import FolderNotFoundException from ._ingestion_in_progress_exception import IngestionInProgressException @@ -23,10 +30,15 @@ from ._operation_not_complete_exception import OperationNotCompleteException from ._secret_missing_error import SecretMissingError from ._unsupported_data_source_exception import UnsupportedDataSourceException +from .datafabric_error_codes import DataFabricErrorCategory __all__ = [ "BaseUrlMissingError", + "DataFabricError", + "DataFabricErrorCategory", + "BatchTransformFailedException", "BatchTransformNotCompleteException", + "ContextGroundingIndexNotFoundError", "EnrichedException", "ExtractedErrorInfo", "FolderNotFoundException", diff --git a/packages/uipath-platform/src/uipath/platform/errors/_batch_transform_failed_exception.py b/packages/uipath-platform/src/uipath/platform/errors/_batch_transform_failed_exception.py new file mode 100644 index 000000000..67088b0f1 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/errors/_batch_transform_failed_exception.py @@ -0,0 +1,10 @@ +class BatchTransformFailedException(Exception): + """Raised when a batch transform has failed. + + This exception is raised when a batch transform task has completed + with a failed status, as opposed to still being in progress. + """ + + def __init__(self, batch_transform_id: str): + self.message = f"Batch transform '{batch_transform_id}' failed." + super().__init__(self.message) diff --git a/packages/uipath-platform/src/uipath/platform/errors/_context_grounding_index_not_found_exception.py b/packages/uipath-platform/src/uipath/platform/errors/_context_grounding_index_not_found_exception.py new file mode 100644 index 000000000..653be92e8 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/errors/_context_grounding_index_not_found_exception.py @@ -0,0 +1,13 @@ +from typing import Optional + + +class ContextGroundingIndexNotFoundError(Exception): + """Raised when a context grounding index cannot be resolved by name.""" + + def __init__(self, index_name: Optional[str] = None): + self.index_name = index_name + if index_name: + self.message = f"ContextGroundingIndex '{index_name}' not found" + else: + self.message = "ContextGroundingIndex not found" + super().__init__(self.message) diff --git a/packages/uipath-platform/src/uipath/platform/errors/_datafabric_error.py b/packages/uipath-platform/src/uipath/platform/errors/_datafabric_error.py new file mode 100644 index 000000000..e697ac164 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/errors/_datafabric_error.py @@ -0,0 +1,109 @@ +"""Data Fabric query engine error classification. + +Maps error codes from the DF query engine invoking the "query_execute" endpoint to actionable +categories so that callers (e.g. the agent SQL sub-graph) can decide +whether to retry, ask the LLM to fix the SQL, or surface an infra error. + +The server error response JSON has the shape: + {"error": "", "code": "", "traceId": ""} +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Callable, TypeVar + +from ._extractors._helpers import extract_service_prefix +from .datafabric_error_codes import ( + _QUERY_ENTITY_RECORDS_ERROR_CODES, + DataFabricErrorCategory, + classify_error_code, +) + +if TYPE_CHECKING: + from ._enriched_exception import EnrichedException + + +TCallable = TypeVar("TCallable", bound=Callable[..., Any]) + + +def attach_datafabric_error_mapping( + method_name: str, +) -> Callable[[TCallable], TCallable]: + """Attach Data Fabric error metadata to a query method.""" + + def decorator(func: TCallable) -> TCallable: + func.__uipath_datafabric_method__ = method_name # type: ignore[attr-defined] + func.__uipath_datafabric_error_codes__ = ( # type: ignore[attr-defined] + _QUERY_ENTITY_RECORDS_ERROR_CODES + ) + return func + + return decorator + + +@dataclass(frozen=True) +class DataFabricError: + """Structured error parsed from a DF query engine response.""" + + code: str | None + message: str | None + trace_id: str | None + category: DataFabricErrorCategory + + @property + def is_retryable(self) -> bool: + return self.category == DataFabricErrorCategory.RETRYABLE + + @property + def is_bad_sql(self) -> bool: + return self.category == DataFabricErrorCategory.BAD_SQL + + @staticmethod + def from_enriched_exception(exc: EnrichedException) -> DataFabricError | None: + """Extract a DataFabricError from an EnrichedException, if applicable. + + Returns None if the exception is not from a Data Fabric endpoint. + """ + if extract_service_prefix(exc.url) != "datafabric_": + return None + + info = exc.error_info + code = info.error_code if info else None + message = info.message if info else None + trace_id = info.trace_id if info else None + + return DataFabricError( + code=code, + message=message, + trace_id=trace_id, + category=classify_error_code(code), + ) + + @staticmethod + def from_response_body(body: dict[str, Any]) -> DataFabricError: + """Parse a DataFabricError directly from a response body dict.""" + raw_code = body.get("code") + code = ( + str(raw_code) + if raw_code is not None and not isinstance(raw_code, (dict, list)) + else None + ) + message = body.get("error") + if not isinstance(message, str): + message = ( + body.get("message") if isinstance(body.get("message"), str) else None + ) + trace_id = body.get("traceId") + if not isinstance(trace_id, str): + trace_id = ( + body.get("requestId") + if isinstance(body.get("requestId"), str) + else None + ) + return DataFabricError( + code=code, + message=message, + trace_id=trace_id, + category=classify_error_code(code), + ) diff --git a/packages/uipath-platform/src/uipath/platform/errors/_extractors/_datafabric.py b/packages/uipath-platform/src/uipath/platform/errors/_extractors/_datafabric.py new file mode 100644 index 000000000..57682a24f --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/errors/_extractors/_datafabric.py @@ -0,0 +1,21 @@ +"""Data Fabric query engine error payload extractor. + +DF returns: {"error": "", "code": "", "traceId": ""} +""" + +from typing import Any + +from .._enriched_exception import ExtractedErrorInfo +from ._helpers import get_str_field, get_typed_field + + +def extract_datafabric(body: dict[str, Any]) -> ExtractedErrorInfo: + message = get_typed_field(body, str, "error", "message") + error_code = get_str_field(body, "code", "errorCode") + trace_id = get_typed_field(body, str, "traceId", "requestId") + + return ExtractedErrorInfo( + message=message, + error_code=error_code, + trace_id=trace_id, + ) diff --git a/packages/uipath-platform/src/uipath/platform/errors/_extractors/_router.py b/packages/uipath-platform/src/uipath/platform/errors/_extractors/_router.py index 37369113b..901dbc089 100644 --- a/packages/uipath-platform/src/uipath/platform/errors/_extractors/_router.py +++ b/packages/uipath-platform/src/uipath/platform/errors/_extractors/_router.py @@ -14,6 +14,7 @@ from .._enriched_exception import ExtractedErrorInfo from ._agenthub import extract_agenthub from ._apps import extract_apps +from ._datafabric import extract_datafabric from ._elements import extract_elements from ._generic import extract_generic from ._helpers import extract_service_prefix, is_llm_path @@ -29,6 +30,7 @@ "agenthub_": extract_agenthub, "agentsruntime_": extract_agenthub, "apps_": extract_apps, + "datafabric_": extract_datafabric, "elements_": extract_elements, "llmopstenant_": extract_llmops, } diff --git a/packages/uipath-platform/src/uipath/platform/errors/datafabric_error_codes.py b/packages/uipath-platform/src/uipath/platform/errors/datafabric_error_codes.py new file mode 100644 index 000000000..81130db95 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/errors/datafabric_error_codes.py @@ -0,0 +1,69 @@ +"""Data Fabric query-engine error code constants.""" + +from __future__ import annotations + +from enum import Enum + +_RETRYABLE_CODES: frozenset[str] = frozenset( + { + "EXECUTION_TIMEOUT", + "SQLITE_BUSY", + "EXECUTION_INTERRUPTED", + } +) + +_BAD_SQL_CODES: frozenset[str] = frozenset( + { + "SQL_PARSING", + "SQL_VALIDATION", + } +) + +_INFRASTRUCTURE_CODES: frozenset[str] = frozenset( + { + "SQLITE_MEMORY_FULL", + "EPHEMERAL_STORAGE_ERROR", + "INTERNAL_ERROR", + "FQS_ERROR", + } +) + +_DATA_ISSUE_CODES: frozenset[str] = frozenset( + { + "FRAGMENT_EXECUTION_FAILURE", + "CONTEXT_CREATION", + "UNKNOWN_ENTITY", + "EXECUTION_ERROR", + "RESULT_TOO_LARGE", + } +) + +_QUERY_ENTITY_RECORDS_ERROR_CODES: frozenset[str] = frozenset( + {*_RETRYABLE_CODES, *_BAD_SQL_CODES, *_INFRASTRUCTURE_CODES, *_DATA_ISSUE_CODES} +) + + +class DataFabricErrorCategory(str, Enum): + """Actionable error category for Data Fabric query failures.""" + + RETRYABLE = "retryable" + BAD_SQL = "bad_sql" + INFRASTRUCTURE = "infrastructure" + DATA_ISSUE = "data_issue" + UNKNOWN = "unknown" + + +def classify_error_code(code: str | None) -> DataFabricErrorCategory: + """Classify a DF error code string into an actionable category.""" + if not code: + return DataFabricErrorCategory.UNKNOWN + upper = code.upper() + if upper in _RETRYABLE_CODES: + return DataFabricErrorCategory.RETRYABLE + if upper in _BAD_SQL_CODES: + return DataFabricErrorCategory.BAD_SQL + if upper in _INFRASTRUCTURE_CODES: + return DataFabricErrorCategory.INFRASTRUCTURE + if upper in _DATA_ISSUE_CODES: + return DataFabricErrorCategory.DATA_ISSUE + return DataFabricErrorCategory.UNKNOWN diff --git a/packages/uipath-platform/src/uipath/platform/external_applications/__init__.py b/packages/uipath-platform/src/uipath/platform/external_applications/__init__.py new file mode 100644 index 000000000..612d131c0 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/external_applications/__init__.py @@ -0,0 +1,5 @@ +"""UiPath External Applications.""" + +from ._external_application_service import ExternalApplicationService + +__all__ = ["ExternalApplicationService"] diff --git a/packages/uipath-platform/src/uipath/platform/common/_external_application_service.py b/packages/uipath-platform/src/uipath/platform/external_applications/_external_application_service.py similarity index 63% rename from packages/uipath-platform/src/uipath/platform/common/_external_application_service.py rename to packages/uipath-platform/src/uipath/platform/external_applications/_external_application_service.py index 5c27ab779..6fac449d4 100644 --- a/packages/uipath-platform/src/uipath/platform/common/_external_application_service.py +++ b/packages/uipath-platform/src/uipath/platform/external_applications/_external_application_service.py @@ -1,3 +1,5 @@ +from collections.abc import Iterator +from contextlib import contextmanager from os import environ as env from typing import Optional from urllib.parse import urlparse @@ -5,10 +7,11 @@ import httpx from httpx import HTTPStatusError +from uipath.platform.constants import ENV_BASE_URL + +from ..common.auth import TokenData from ..errors import EnrichedException from ..identity import IdentityService -from .auth import TokenData -from .constants import ENV_BASE_URL class ExternalApplicationService: @@ -71,6 +74,31 @@ def _extract_environment_from_base_url(self, base_url: str) -> str: # Default to cloud if parsing fails return "cloud" + @contextmanager + def _translate_auth_errors(self) -> Iterator[None]: + """Translate token acquisition failures into authentication errors.""" + try: + yield + except HTTPStatusError as e: + match e.response.status_code: + case 400: + message = "Invalid client credentials or request parameters." + case 401: + message = "Unauthorized: Invalid client credentials." + case _: + message = f"Authentication failed with unexpected status: {e.response.status_code}" + raise EnrichedException( + HTTPStatusError( + message=message, + request=e.request, + response=e.response, + ) + ) from e + except httpx.RequestError as e: + raise Exception(f"Network error during authentication: {e}") from e + except Exception as e: + raise Exception(f"Unexpected error during authentication: {e}") from e + def get_token_data( self, client_id: str, client_secret: str, scope: Optional[str] = "OR.Execution" ) -> TokenData: @@ -84,42 +112,12 @@ def get_token_data( Returns: Token data if successful """ - try: + with self._translate_auth_errors(): return self._identity_service.get_client_credentials_token( client_id=client_id, client_secret=client_secret, scope=scope, ) - except HTTPStatusError as e: - match e.response.status_code: - case 400: - raise EnrichedException( - HTTPStatusError( - message="Invalid client credentials or request parameters.", - request=e.request, - response=e.response, - ) - ) from e - case 401: - raise EnrichedException( - HTTPStatusError( - message="Unauthorized: Invalid client credentials.", - request=e.request, - response=e.response, - ) - ) from e - case _: - raise EnrichedException( - HTTPStatusError( - message=f"Authentication failed with unexpected status: {e.response.status_code}", - request=e.request, - response=e.response, - ) - ) from e - except httpx.RequestError as e: - raise Exception(f"Network error during authentication: {e}") from e - except Exception as e: - raise Exception(f"Unexpected error during authentication: {e}") from e async def get_token_data_async( self, client_id: str, client_secret: str, scope: Optional[str] = "OR.Execution" @@ -134,39 +132,9 @@ async def get_token_data_async( Returns: Token data if successful """ - try: + with self._translate_auth_errors(): return await self._identity_service.get_client_credentials_token_async( client_id=client_id, client_secret=client_secret, scope=scope, ) - except HTTPStatusError as e: - match e.response.status_code: - case 400: - raise EnrichedException( - HTTPStatusError( - message="Invalid client credentials or request parameters.", - request=e.request, - response=e.response, - ) - ) from e - case 401: - raise EnrichedException( - HTTPStatusError( - message="Unauthorized: Invalid client credentials.", - request=e.request, - response=e.response, - ) - ) from e - case _: - raise EnrichedException( - HTTPStatusError( - message=f"Authentication failed with unexpected status: {e.response.status_code}", - request=e.request, - response=e.response, - ) - ) from e - except httpx.RequestError as e: - raise Exception(f"Network error during authentication: {e}") from e - except Exception as e: - raise Exception(f"Unexpected error during authentication: {e}") from e diff --git a/packages/uipath-platform/src/uipath/platform/governance/__init__.py b/packages/uipath-platform/src/uipath/platform/governance/__init__.py new file mode 100644 index 000000000..1d9bdf7ee --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/governance/__init__.py @@ -0,0 +1,29 @@ +"""Governance services for the UiPath Platform. + +Exposes the agenticgovernance_ ingress: tenant-controlled policy packs +served centrally so policy decisions can change without redeploying +agents. +""" + +from ._governance_provider import UiPathPlatformGovernanceProvider +from ._governance_service import GovernanceService +from .compensate import FiredRule, GovernRequest +from .policy import PolicyContext, PolicyResponse + +# ``_live_track_event_dispatcher.LiveTrackEventDispatcher`` is intentionally +# **not** re-exported. It is host-wiring glue (the runtime sink's +# non-blocking ``track_event`` adapter), not a customer-facing API. +# Internal callers import it via the explicit private path: +# +# from uipath.platform.governance._live_track_event_dispatcher import ( +# LiveTrackEventDispatcher, +# ) + +__all__ = [ + "FiredRule", + "GovernRequest", + "GovernanceService", + "PolicyContext", + "PolicyResponse", + "UiPathPlatformGovernanceProvider", +] diff --git a/packages/uipath-platform/src/uipath/platform/governance/_governance_provider.py b/packages/uipath-platform/src/uipath/platform/governance/_governance_provider.py new file mode 100644 index 000000000..1b336ff0e --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/governance/_governance_provider.py @@ -0,0 +1,114 @@ +"""Platform-backed implementation of the core governance provider protocols. + +Thin adapter around :class:`GovernanceService` that exposes only the +methods required by +:class:`uipath.core.governance.GovernancePolicyProvider` and +:class:`uipath.core.governance.GovernanceCompensationProvider`. + +Wrap an existing :class:`GovernanceService` (e.g. +``UiPathPlatformGovernanceProvider(service=UiPath().governance)``) or +pass ``config``/``execution_context`` to construct one inline. +""" + +from __future__ import annotations + +from typing import Any + +from uipath.core.governance import GovernRequest, PolicyContext, PolicyResponse + +from ..common._config import UiPathApiConfig +from ..common._execution_context import UiPathExecutionContext +from ._governance_service import GovernanceService + + +class UiPathPlatformGovernanceProvider: + """Platform-backed governance provider. + + Implements both + :class:`uipath.core.governance.GovernancePolicyProvider` and + :class:`uipath.core.governance.GovernanceCompensationProvider` by + delegating to :class:`GovernanceService`. + + Args: + service: Existing :class:`GovernanceService` to delegate to. + Useful for tests and for sharing an SDK service across + consumers. When omitted, a fresh service is built from the + ``config`` and ``execution_context`` kwargs. + config: Required when ``service`` is not supplied. + execution_context: Required when ``service`` is not supplied. + """ + + def __init__( + self, + service: GovernanceService | None = None, + *, + config: UiPathApiConfig | None = None, + execution_context: UiPathExecutionContext | None = None, + ) -> None: + if service is None: + if config is None or execution_context is None: + raise ValueError( + "UiPathPlatformGovernanceProvider requires either a " + "GovernanceService instance or both config and " + "execution_context." + ) + service = GovernanceService( + config=config, execution_context=execution_context + ) + self._service = service + + @property + def service(self) -> GovernanceService: + """The underlying :class:`GovernanceService` instance.""" + return self._service + + # ── GovernancePolicyProvider ───────────────────────────────────── + + def get_policy(self, context: PolicyContext) -> PolicyResponse: + """Fetch the policy pack — delegates to ``GovernanceService``.""" + return self._service.get_policy(context) + + async def get_policy_async(self, context: PolicyContext) -> PolicyResponse: + """Async variant of :meth:`get_policy`.""" + return await self._service.get_policy_async(context) + + # ── GovernanceCompensationProvider ─────────────────────────────── + + def compensate(self, request: GovernRequest) -> None: + """Fire the compensating ``/runtime/govern`` POST.""" + self._service._compensate(request) + + async def compensate_async(self, request: GovernRequest) -> None: + """Async variant of :meth:`compensate`.""" + await self._service._compensate_async(request) + + # ── Custom telemetry events ────────────────────────────────────── + + def track_event( + self, + *, + event_name: str, + data: dict[str, Any] | None = None, + operation_id: str | None = None, + ) -> None: + """Record a custom telemetry event — delegates to ``GovernanceService``. + + See :meth:`GovernanceService._track_event` for parameter + semantics — in particular, the ``operation_id`` → trace-id + fallback. + """ + self._service._track_event( + event_name=event_name, data=data, operation_id=operation_id + ) + + async def track_event_async( + self, + *, + event_name: str, + data: dict[str, Any] | None = None, + operation_id: str | None = None, + ) -> None: + """Async variant of :meth:`track_event`.""" + await self._service._track_event_async( + event_name=event_name, data=data, operation_id=operation_id + ) diff --git a/packages/uipath-platform/src/uipath/platform/governance/_governance_service.py b/packages/uipath-platform/src/uipath/platform/governance/_governance_service.py new file mode 100644 index 000000000..3546517c7 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/governance/_governance_service.py @@ -0,0 +1,488 @@ +"""Service for the ``agenticgovernance_`` ingress. + +Wraps the governance backend endpoints UiPath exposes: + +- ``GET /{org}/agenticgovernance_/api/v1/runtime/policy`` — fetch the + tenant-managed policy pack (see :meth:`GovernanceService.retrieve_policy`). +- ``POST /{org}/agenticgovernance_/api/v1/runtime/govern`` — compensating + governance call fired when a ``guardrail_fallback`` rule matches + (see :meth:`GovernanceService.compensate`). + +A third backend endpoint — +``POST /{org}/agenticgovernance_/api/v1/runtime/log`` — emits custom +telemetry events to App Insights. It's reached only through the +internal ``_track_event`` helper, which the runtime adapter +(:class:`UiPathPlatformGovernanceProvider`) calls; not part of the +client-facing service surface. + +Org/tenant scoping is read from :class:`UiPathConfig`; auth, retries, +trace context, and error enrichment come from :class:`BaseService`. +""" + +from typing import Any, Optional + +from uipath.core import traced +from uipath.core.governance import ( + FiredRule, + GovernRequest, + PolicyContext, + PolicyResponse, +) + +from uipath.platform.constants import HEADER_INTERNAL_TENANT_ID + +from ..common._base_service import BaseService, resolve_trace_id +from ..common._config import UiPathConfig +from ..common._service_url_overrides import ( + inject_routing_headers, + resolve_service_url, +) + +# The agenticgovernance_ ingress lives at a separate org-scoped path that +# uses the organization UUID (not the slug exposed by ``UIPATH_URL``). +GOVERNANCE_SERVICE_PREFIX = "agenticgovernance_" +POLICY_API_PATH = "api/v1/runtime/policy" +GOVERN_API_PATH = "api/v1/runtime/govern" +LOG_API_PATH = "api/v1/runtime/log" +AGENT_TYPE_PARAM = "agentType" + +# Caller-set correlation id that becomes the App Insights ``operation_Id`` +# stamped on every customEvent produced by the matching ``/runtime/log`` +# request — see the spec on the platform-side ``postLogHandler``. +HEADER_OPERATION_ID = "x-uipath-operation-id" + + +class GovernanceService(BaseService): + """Service for the agenticgovernance_ ingress. + + Exposes two endpoints: + + - :meth:`retrieve_policy` — GET the tenant-managed policy pack. + - :meth:`compensate` — POST a compensating ``/runtime/govern`` call + so the server can run a disabled centralized guardrail and write + the per-rule LLMOps audit records itself. + + Org and tenant scoping come from :attr:`UiPathConfig.organization_id` + and :attr:`UiPathConfig.tenant_id`; the tenant travels in the + ``x-uipath-internal-tenantid`` header (the URL is org-scoped only). + + !!! info "Version Availability" + This service is available starting from **uipath** version **2.2.13**. + """ + + # ── Policy fetch ───────────────────────────────────────────────── + + @traced(name="governance_retrieve_policy", run_type="uipath") + def retrieve_policy( + self, + *, + is_conversational: Optional[bool] = None, + ) -> PolicyResponse: + """Fetch the governance policy pack for the active org/tenant. + + Args: + is_conversational: When the hosted agent's type is known, + selects the conversational (``True``) or autonomous + (``False``) policy view. ``None`` (default) omits the + ``agentType`` query param so the server applies its + default. + + Returns: + PolicyResponse: ``mode`` and the YAML ``policies`` string. + + Raises: + ValueError: If ``UiPathConfig.organization_id`` or + ``UiPathConfig.tenant_id`` is not set. + EnrichedException: If the backend returns a non-2xx response. + + Examples: + ```python + from uipath.platform import UiPath + + client = UiPath() + response = client.governance.retrieve_policy() + print(response.mode, len(response.policies)) + ``` + """ + url, headers = self._build_org_scoped_request(POLICY_API_PATH) + params = self._policy_params(is_conversational) + response = self.request("GET", url=url, params=params, headers=headers) + return PolicyResponse.model_validate(response.json()) + + @traced(name="governance_retrieve_policy", run_type="uipath") + async def retrieve_policy_async( + self, + *, + is_conversational: Optional[bool] = None, + ) -> PolicyResponse: + """Asynchronously fetch the governance policy pack. + + See :meth:`retrieve_policy` for parameter and return semantics. + """ + url, headers = self._build_org_scoped_request(POLICY_API_PATH) + params = self._policy_params(is_conversational) + response = await self.request_async( + "GET", url=url, params=params, headers=headers + ) + return PolicyResponse.model_validate(response.json()) + + # ── Policy provider adapter (GovernancePolicyProvider protocol) ─ + + def get_policy(self, context: PolicyContext) -> PolicyResponse: + """Fetch the policy pack — :class:`GovernancePolicyProvider` adapter. + + Thin wrapper over :meth:`retrieve_policy` that accepts the + context model the core protocol uses. Lets the runtime consume + governance through :class:`uipath.core.governance.GovernancePolicyProvider` + without importing this module. + """ + return self.retrieve_policy(is_conversational=context.is_conversational) + + async def get_policy_async(self, context: PolicyContext) -> PolicyResponse: + """Async variant of :meth:`get_policy`.""" + return await self.retrieve_policy_async( + is_conversational=context.is_conversational + ) + + # ── Compensating governance call ───────────────────────────────── + + def compensate( + self, + *, + hook: str, + validators: list[str], + rules: list[FiredRule], + data: dict[str, Any], + src_timestamp: str, + agent_name: str, + runtime_id: str, + trace_id: str | None = None, + folder_key: str | None = None, + job_key: str | None = None, + process_key: str | None = None, + reference_id: str | None = None, + agent_version: str | None = None, + ) -> None: + """POST a compensating ``/runtime/govern`` call. + + Fired when a ``guardrail_fallback`` rule matches: the centralized + guardrail is disabled, so the server is asked to run the + guardrail check server-side and write the per-rule LLMOps audit + records bound to the agent's trace. The agent does not inspect + the response body. + + Job-context fields (``folder_key`` / ``job_key`` / + ``process_key`` / ``reference_id`` / ``agent_version``) are + auto-populated from :class:`UiPathConfig` when omitted. + Caller-supplied values — including the empty string — take + precedence. + + Args: + hook: Identifier of the agent hook that fired the rule + (e.g. ``"before_model"``). + validators: Validator names attached to the fired rules. + rules: Each rule that fired — one LLMOps audit record is + written per entry. + data: Hook payload the server replays through the + centralized guardrail. + trace_id: Canonical 32-char hex trace id. Optional — when + ``None`` (default) the service resolves the value + itself at call time via :func:`resolve_trace_id`. + Callers that already hold a resolved id (typically + captured on the hook thread before a background-pool + hop) pass it in to win over the auto-resolve. + src_timestamp: ISO-8601 timestamp on the source side. + agent_name: Agent identifier as known to the platform. + runtime_id: Runtime instance identifier. + folder_key: Override the env-backed folder key. + job_key: Override the env-backed job key. + process_key: Override the env-backed process key. + reference_id: Override the env-backed agent id. + agent_version: Override the env-backed agent version. + + Raises: + ValueError: If ``UiPathConfig.organization_id`` or + ``UiPathConfig.tenant_id`` is not set. + EnrichedException: If the backend returns a non-2xx response. + + Threading: + OpenTelemetry context is thread-local; callers that + background-pool the compensation call must capture the + canonical trace id (via :func:`resolve_trace_id`) on the + hook thread and pass it in explicitly — the auto-resolve + on the worker thread will see a detached context. + """ + self._compensate( + GovernRequest( + hook=hook, + validators=validators, + rules=rules, + data=data, + trace_id=trace_id, + src_timestamp=src_timestamp, + agent_name=agent_name, + runtime_id=runtime_id, + folder_key=folder_key, + job_key=job_key, + process_key=process_key, + reference_id=reference_id, + agent_version=agent_version, + ) + ) + + async def compensate_async( + self, + *, + hook: str, + validators: list[str], + rules: list[FiredRule], + data: dict[str, Any], + src_timestamp: str, + agent_name: str, + runtime_id: str, + trace_id: str | None = None, + folder_key: str | None = None, + job_key: str | None = None, + process_key: str | None = None, + reference_id: str | None = None, + agent_version: str | None = None, + ) -> None: + """Asynchronously POST a compensating ``/runtime/govern`` call. + + See :meth:`compensate` for parameter semantics. + """ + await self._compensate_async( + GovernRequest( + hook=hook, + validators=validators, + rules=rules, + data=data, + trace_id=trace_id, + src_timestamp=src_timestamp, + agent_name=agent_name, + runtime_id=runtime_id, + folder_key=folder_key, + job_key=job_key, + process_key=process_key, + reference_id=reference_id, + agent_version=agent_version, + ) + ) + + # ── Internal worker for GovernRequest-shaped callers ───────────── + + @traced(name="governance_compensate", run_type="uipath") + def _compensate(self, request: GovernRequest) -> None: + """Fire a compensation call from a pre-built :class:`GovernRequest`. + + Internal helper used by the provider adapter + (:class:`uipath.platform.governance.UiPathPlatformGovernanceProvider`) + to satisfy :class:`uipath.core.governance.GovernanceCompensationProvider` + without unpacking the request. The public ergonomic counterpart + is :meth:`compensate`. + + When ``request.trace_id`` is ``None`` the service resolves the + canonical trace id itself via :func:`resolve_trace_id` — same + fallback ``track_event`` uses. Callers that have a resolved + value still pass it in; callers that don't (e.g. the runtime + layer, which intentionally stays env-free) leave it ``None`` + and let the service do the work. + """ + request = self._resolve_request_trace_id(request) + url, headers = self._build_org_scoped_request(GOVERN_API_PATH) + payload = self._build_govern_payload(request) + self.request("POST", url=url, headers=headers, json=payload) + + @traced(name="governance_compensate", run_type="uipath") + async def _compensate_async(self, request: GovernRequest) -> None: + """Async variant of :meth:`_compensate`. + + Same ``trace_id`` self-resolution behavior as the sync variant. + """ + request = self._resolve_request_trace_id(request) + url, headers = self._build_org_scoped_request(GOVERN_API_PATH) + payload = self._build_govern_payload(request) + await self.request_async("POST", url=url, headers=headers, json=payload) + + @staticmethod + def _resolve_request_trace_id(request: GovernRequest) -> GovernRequest: + """Fill ``request.trace_id`` from :func:`resolve_trace_id` when absent. + + Caller-supplied values (including ``""``) win — the runtime + captures on the hook thread (via ``contextvars.copy_context`` + for the background pool) and the resolver here only fires when + the field was left ``None``. + """ + if request.trace_id is not None: + return request + resolved = resolve_trace_id() + if not resolved: + return request + return request.model_copy(update={"trace_id": resolved}) + + # ── Custom telemetry events (internal runtime seam) ────────────── + # + # ``_track_event`` / ``_track_event_async`` are intentionally + # underscore-prefixed: they exist for the runtime adapter + # (:class:`UiPathPlatformGovernanceProvider`) to fire telemetry + # events through the platform's HTTP stack, not as a client-facing + # SDK call. Keeping them off the public surface keeps the auto- + # generated docs (``mkdocs`` + ``mkdocstrings``) focused on the + # endpoints customers consume directly (``retrieve_policy`` / + # ``compensate``). + + def _track_event( + self, + *, + event_name: str, + data: dict[str, Any] | None = None, + operation_id: str | None = None, + ) -> None: + """POST a custom telemetry event to ``/runtime/log``. + + Internal seam — the runtime adapter + (:class:`UiPathPlatformGovernanceProvider`) calls this to emit + governance audit events through the platform's HTTP stack. + The server forwards the event to App Insights as a + ``customEvents`` row; account / tenant / organization are + stamped server-side from the gateway headers and JWT. + + Args: + event_name: Non-empty event name — becomes the App Insights + row ``name``. The platform redactor runs over this before + it reaches the sink. + data: Optional properties flattened into the event. Non-dict + values are dropped server-side. + operation_id: Optional correlation id forwarded as the + ``x-uipath-operation-id`` header. When omitted, falls + back to :func:`resolve_trace_id` so events emitted from + the same agent trace share an ``operation_Id`` and are + queryable together in KQL. When neither is available, + the header is omitted and App Insights generates its + own id per event. + + Raises: + ValueError: If ``event_name`` is empty / whitespace-only, or + if ``UiPathConfig.organization_id`` / + ``UiPathConfig.tenant_id`` is not set. + EnrichedException: If the backend returns a non-2xx response. + """ + self._validate_event_name(event_name) + url, headers = self._build_org_scoped_request(LOG_API_PATH) + resolved_op_id = operation_id or resolve_trace_id() + if resolved_op_id: + headers[HEADER_OPERATION_ID] = resolved_op_id + payload: dict[str, Any] = {"eventName": event_name} + if data is not None: + payload["data"] = data + self.request("POST", url=url, headers=headers, json=payload) + + async def _track_event_async( + self, + *, + event_name: str, + data: dict[str, Any] | None = None, + operation_id: str | None = None, + ) -> None: + """Async variant of :meth:`_track_event`. Internal seam.""" + self._validate_event_name(event_name) + url, headers = self._build_org_scoped_request(LOG_API_PATH) + resolved_op_id = operation_id or resolve_trace_id() + if resolved_op_id: + headers[HEADER_OPERATION_ID] = resolved_op_id + payload: dict[str, Any] = {"eventName": event_name} + if data is not None: + payload["data"] = data + await self.request_async("POST", url=url, headers=headers, json=payload) + + @staticmethod + def _validate_event_name(event_name: str) -> None: + """Reject empty/whitespace-only event names client-side. + + The platform's ``/runtime/log`` handler rejects these with a + 4xx; failing fast here gives the caller a clearer error and + avoids the round trip. + """ + if not event_name or not event_name.strip(): + raise ValueError("event_name must be a non-empty string.") + + # ── Internals ──────────────────────────────────────────────────── + + def _build_org_scoped_request(self, path: str) -> tuple[str, dict[str, str]]: + """Compose the agenticgovernance_ URL and the tenant header. + + Both governance endpoints share the same URL shape + (``{origin}/{org_id_uuid}/agenticgovernance_/{path}``) and the + same ``x-uipath-internal-tenantid`` header — neither matches + ``UiPathUrl.scope_url`` (slug-based), so the URL is composed + directly here. + + Honors ``UIPATH_SERVICE_URL_AGENTICGOVERNANCE`` for local dev: + when set, redirects to the override and injects routing headers + so the local server sees what the platform router would have + carried. ``BaseService.request`` does this same dance for paths + that fit ``scope_url``; the org-UUID-in-path shape forces us to + run it ourselves before composing the absolute URL. + """ + organization_id = UiPathConfig.organization_id + if not organization_id: + raise ValueError( + "Governance call requires UIPATH_ORGANIZATION_ID " + "to be set in the environment." + ) + tenant_id = UiPathConfig.tenant_id + if not tenant_id: + raise ValueError( + "Governance call requires UIPATH_TENANT_ID " + "to be set in the environment." + ) + + override = resolve_service_url(f"{GOVERNANCE_SERVICE_PREFIX}/{path}") + if override: + headers: dict[str, str] = {} + inject_routing_headers(headers) + return override, headers + + url = ( + f"{self._url.base_url}/{organization_id}/{GOVERNANCE_SERVICE_PREFIX}/{path}" + ) + return url, {HEADER_INTERNAL_TENANT_ID: tenant_id} + + @staticmethod + def _policy_params(is_conversational: Optional[bool]) -> dict[str, str]: + if is_conversational is None: + return {} + return { + AGENT_TYPE_PARAM: "conversational" if is_conversational else "autonomous" + } + + @staticmethod + def _build_govern_payload(request: GovernRequest) -> dict[str, Any]: + """Serialize the request and fill missing job-context from UiPathConfig. + + Auto-fill resolution order for each job-context field: caller + value > ``UiPathConfig`` (env-var-backed) > omit. + + ``model_dump(exclude_none=True)`` already drops fields the caller + left ``None``, so key presence — not truthiness — is the right + "was it supplied?" signal: a caller-supplied empty string is + still a caller value and must not be overridden by the env. + """ + payload = request.model_dump(by_alias=True, exclude_none=True) + for wire_key, config_attr in _JOB_CONTEXT_FIELDS: + if wire_key in payload: + continue + value = getattr(UiPathConfig, config_attr, None) + if value: + payload[wire_key] = value + return payload + + +# Wire-key → UiPathConfig attribute, for compensation payload auto-fill. +_JOB_CONTEXT_FIELDS: tuple[tuple[str, str], ...] = ( + ("folderKey", "folder_key"), + ("jobKey", "job_key"), + ("processKey", "process_uuid"), + ("referenceId", "agent_id"), + ("agentVersion", "process_version"), +) diff --git a/packages/uipath-platform/src/uipath/platform/governance/_live_track_event_dispatcher.py b/packages/uipath-platform/src/uipath/platform/governance/_live_track_event_dispatcher.py new file mode 100644 index 000000000..432fd91c5 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/governance/_live_track_event_dispatcher.py @@ -0,0 +1,287 @@ +"""Non-blocking dispatcher for governance track-event telemetry. + +Wraps :meth:`UiPathPlatformGovernanceProvider.track_event_async` on a +private background ``asyncio`` event loop so sync callers can fire +telemetry events without blocking on the underlying ``POST /runtime/log`` +HTTP round-trip. + +:meth:`LiveTrackEventDispatcher.dispatch` is a sync fire-and-forget +method that mirrors the kwargs of ``track_event_async``. Internally it +schedules the async HTTP call onto a dedicated background loop, so the +calling thread never blocks on network I/O and the underlying HTTP call +remains async end-to-end. + +Design notes: + +- **Async HTTP inside, sync interface outside.** ``dispatch`` is a + sync function. Internally it enqueues a coroutine that awaits + ``provider.track_event_async``. + +- **Loop affinity.** ``httpx.AsyncClient`` lazy-binds its connection + pool to the first event loop that awaits on it. This dispatcher + assumes it owns the provider's async HTTP path — nothing else in + the process should await ``track_event_async`` (or any other + ``*_async`` method on the same underlying service) on a *different* + loop. See "one dispatcher per provider" below. + +- **Backpressure.** A ``BoundedSemaphore`` caps in-flight coroutines; + submissions that exceed the cap are dropped with a warning so + memory stays bounded when the backend is slow. + +- **Fire-and-forget contract.** Coroutine exceptions are observed on + the returned ``concurrent.futures.Future`` (to suppress asyncio's + "exception was never retrieved" warning) and logged at debug — they + cannot reach the caller because ``dispatch`` returns before the + coroutine runs. + +One dispatcher per provider. The dispatcher's background loop must be +the only loop that awaits the provider's async methods. +""" + +from __future__ import annotations + +import asyncio +import concurrent.futures +import logging +import threading +from typing import Any + +from ._governance_provider import UiPathPlatformGovernanceProvider + +logger = logging.getLogger(__name__) + + +class LiveTrackEventDispatcher: + """Non-blocking sync adapter around ``provider.track_event_async``. + + Schedules governance telemetry events on a private background + ``asyncio`` loop so the calling thread is never blocked on the + platform's ``/runtime/log`` HTTP call — and the HTTP call itself + is awaited (not run on a sync thread pool). + + .. code-block:: python + + provider = UiPathPlatformGovernanceProvider(config=..., execution_context=...) + dispatcher = LiveTrackEventDispatcher(provider) + dispatcher.dispatch(event_name="agent.started") + # ... + dispatcher.shutdown() # at process exit + + ``dispatch`` has the same kwargs as + :meth:`UiPathPlatformGovernanceProvider.track_event_async` so it is + a drop-in sync callable for anywhere the async method would go. + """ + + _DEFAULT_MAX_INFLIGHT = 40 + + def __init__( + self, + provider: UiPathPlatformGovernanceProvider, + *, + max_inflight: int = _DEFAULT_MAX_INFLIGHT, + ) -> None: + """Construct a dispatcher bound to one provider. + + Starts a daemon thread that runs a private ``asyncio`` event + loop. All HTTP awaits happen on that loop; nothing else in the + process should await the provider's async methods on a + different loop (see the module docstring on loop affinity). + + Args: + provider: The platform governance provider whose + ``track_event_async`` will be awaited on the background + loop. + max_inflight: Cap on concurrent in-flight coroutines. When + exceeded, further ``dispatch`` calls are dropped with a + warning so memory stays bounded under a slow backend. + Default 40 is sized for a bursty-but-not-sustained + event stream. + """ + self._provider = provider + self._max_inflight = max_inflight + self._inflight = threading.BoundedSemaphore(max_inflight) + self._shutdown_event = threading.Event() + self._futures_lock = threading.Lock() + self._futures: set[concurrent.futures.Future[None]] = set() + + self._loop = asyncio.new_event_loop() + self._loop_ready = threading.Event() + self._loop_thread = threading.Thread( + target=self._run_loop, + name="governance-track-event-loop", + daemon=True, + ) + self._loop_thread.start() + # Block until the loop is running so the first ``dispatch`` cannot + # race with startup and hit "loop not running" errors. + self._loop_ready.wait() + + def _run_loop(self) -> None: + """Body of the background loop thread — runs until ``shutdown``.""" + asyncio.set_event_loop(self._loop) + self._loop_ready.set() + try: + self._loop.run_forever() + finally: + # After ``run_forever`` returns (from ``stop()``), any tasks + # that were still awaiting mid-flight need to be cancelled + # and finalized before the loop can close cleanly. Without + # this, ``loop.close()`` warns "Task was destroyed but it is + # pending" for every unfinished awaiter. + try: + pending = asyncio.all_tasks(self._loop) + for task in pending: + task.cancel() + if pending: + self._loop.run_until_complete( + asyncio.gather(*pending, return_exceptions=True) + ) + except Exception as exc: # noqa: BLE001 - teardown must not raise + logger.debug("Loop cleanup swallowed exception: %s", exc) + finally: + try: + self._loop.close() + except Exception as exc: # noqa: BLE001 + logger.debug("Loop close swallowed exception: %s", exc) + + def dispatch( + self, + *, + event_name: str, + data: dict[str, Any] | None = None, + operation_id: str | None = None, + ) -> None: + """Schedule a track-event call on the background loop — returns immediately. + + The kwargs mirror + :meth:`UiPathPlatformGovernanceProvider.track_event_async` so + this method is a drop-in sync callable for the async provider + method. + + Failure modes — all silent, never raised to the caller: + + - **Post-shutdown**: dispatch after :meth:`shutdown` returns + silently; the provider is not called. + - **Saturated in-flight cap**: when ``max_inflight`` coroutines + are already scheduled, the call is dropped with a warning. + Telemetry must never grow memory without bound when the + backend is slow. + - **Loop unavailable**: ``asyncio.run_coroutine_threadsafe`` + raises ``RuntimeError`` if the loop is stopped/closed + (late-firing atexit path); the dispatcher rolls back the + semaphore slot, closes the coroutine, and logs at debug. + - **Coroutine exception**: the provider's HTTP call may raise + for any reason (serialization, 5xx, transport). ``_run`` + catches, logs at debug with ``exc_info=True``, and the + done-callback observes the future to suppress asyncio's + "exception was never retrieved" warning. + """ + if self._shutdown_event.is_set(): + logger.debug( + "Dispatcher shut down; dropping track_event (event_name=%s)", + event_name, + ) + return + + if not self._inflight.acquire(blocking=False): + logger.warning( + "Telemetry pool saturated (>%d in flight); dropping track_event " + "(event_name=%s)", + self._max_inflight, + event_name, + ) + return + + coro = self._run(event_name=event_name, data=data, operation_id=operation_id) + try: + future = asyncio.run_coroutine_threadsafe(coro, self._loop) + except RuntimeError as exc: + # Loop is stopped/closed — release the slot we took and + # close the coroutine so it doesn't warn at GC time. + coro.close() + self._inflight.release() + logger.debug( + "Telemetry loop unavailable (event_name=%s): %s", + event_name, + exc, + ) + return + + with self._futures_lock: + self._futures.add(future) + future.add_done_callback(self._on_future_done) + + async def _run( + self, + *, + event_name: str, + data: dict[str, Any] | None, + operation_id: str | None, + ) -> None: + """Coroutine body — the async HTTP call itself.""" + try: + await self._provider.track_event_async( + event_name=event_name, + data=data, + operation_id=operation_id, + ) + except Exception as exc: # noqa: BLE001 - fire-and-forget contract + logger.debug("Failed to dispatch track_event: %s", exc, exc_info=True) + + def _on_future_done(self, future: concurrent.futures.Future[None]) -> None: + """Observe the future, drop it from the pending set, release the slot. + + Uses ``future.exception()`` to observe the outcome so asyncio + doesn't warn "exception was never retrieved" at GC time. + ``concurrent.futures.Future.exception()`` *raises* + ``CancelledError`` when the future was cancelled (the observe- + without-raise semantics apply only to :class:`asyncio.Future`, + not this ``concurrent.futures`` type), so the observation is + wrapped in a targeted catch. The accounting — semaphore release + and pending-set discard — runs in ``finally`` so success, + failure, and cancellation all clean up correctly. + """ + try: + future.exception() + except concurrent.futures.CancelledError: + # Cancellation during shutdown is expected; the underlying + # coroutine's own exception (if any) was already logged by + # ``_run``. + pass + finally: + with self._futures_lock: + self._futures.discard(future) + self._inflight.release() + + def shutdown(self, *, wait: bool = True, timeout: float = 30.0) -> None: + """Stop accepting new submissions; optionally drain pending, then stop the loop. + + Call at process exit to avoid losing in-flight telemetry. + Safe to call more than once — subsequent calls are no-ops. + + Args: + wait: When ``True`` (default), block until pending + coroutines finish (bounded by ``timeout``) before + stopping the loop. When ``False``, stop immediately; + in-flight coroutines are cancelled by the loop's + teardown path. + timeout: Maximum seconds to wait for pending coroutines + when ``wait=True``. Coroutines still in flight after + the timeout are cancelled by loop teardown. + """ + if self._shutdown_event.is_set(): + return + self._shutdown_event.set() + + if wait: + with self._futures_lock: + pending = list(self._futures) + if pending: + concurrent.futures.wait(pending, timeout=timeout) + + try: + self._loop.call_soon_threadsafe(self._loop.stop) + except RuntimeError: + # Loop already stopped. + pass + self._loop_thread.join(timeout=5.0) diff --git a/packages/uipath-platform/src/uipath/platform/governance/compensate.py b/packages/uipath-platform/src/uipath/platform/governance/compensate.py new file mode 100644 index 000000000..bad4845f9 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/governance/compensate.py @@ -0,0 +1,10 @@ +"""Re-exports of compensation models from :mod:`uipath.core.governance`. + +The wire-shape models live in ``uipath-core`` so the runtime can depend on +the protocol contract without importing ``uipath-platform``. This module +keeps the existing ``uipath.platform.governance`` import paths working. +""" + +from uipath.core.governance import FiredRule, GovernRequest + +__all__ = ["FiredRule", "GovernRequest"] diff --git a/packages/uipath-platform/src/uipath/platform/governance/policy.py b/packages/uipath-platform/src/uipath/platform/governance/policy.py new file mode 100644 index 000000000..27de1c9e7 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/governance/policy.py @@ -0,0 +1,10 @@ +"""Re-exports of governance policy models from :mod:`uipath.core.governance`. + +The wire-shape models live in ``uipath-core`` so the runtime can depend on +the protocol contract without importing ``uipath-platform``. This module +keeps the existing ``uipath.platform.governance`` import paths working. +""" + +from uipath.core.governance import PolicyContext, PolicyResponse + +__all__ = ["PolicyContext", "PolicyResponse"] diff --git a/packages/uipath-platform/src/uipath/platform/guardrails/__init__.py b/packages/uipath-platform/src/uipath/platform/guardrails/__init__.py index ffab74581..0f6a16209 100644 --- a/packages/uipath-platform/src/uipath/platform/guardrails/__init__.py +++ b/packages/uipath-platform/src/uipath/platform/guardrails/__init__.py @@ -14,7 +14,37 @@ ) from ._guardrails_service import GuardrailsService +from .decorators import ( + BlockAction, + BuiltInGuardrailValidator, + ByoValidator, + CustomGuardrailValidator, + CustomValidator, + GuardrailAction, + GuardrailBlockException, + GuardrailExclude, + GuardrailExecutionStage, + GuardrailTargetAdapter, + GuardrailValidatorBase, + HarmfulContentEntity, + HarmfulContentEntityType, + HarmfulContentValidator, + IntellectualPropertyEntityType, + IntellectualPropertyValidator, + LLMAsJudgeValidator, + LogAction, + LoggingSeverityLevel, + PIIDetectionEntity, + PIIDetectionEntityType, + PIIValidator, + PromptInjectionValidator, + RuleFunction, + UserPromptAttacksValidator, + guardrail, + register_guardrail_adapter, +) from .guardrails import ( + BYO_VALIDATOR_TYPE, BuiltInValidatorGuardrail, EnumListParameterValue, GuardrailType, @@ -22,7 +52,10 @@ ) __all__ = [ + # Service "GuardrailsService", + # Guardrail models + "BYO_VALIDATOR_TYPE", "BuiltInValidatorGuardrail", "GuardrailType", "GuardrailValidationResultType", @@ -33,4 +66,32 @@ "GuardrailValidationResult", "EnumListParameterValue", "MapEnumParameterValue", + # Decorator framework + "guardrail", + "GuardrailValidatorBase", + "BuiltInGuardrailValidator", + "ByoValidator", + "CustomGuardrailValidator", + "HarmfulContentValidator", + "IntellectualPropertyValidator", + "LLMAsJudgeValidator", + "PIIValidator", + "PromptInjectionValidator", + "UserPromptAttacksValidator", + "CustomValidator", + "RuleFunction", + "HarmfulContentEntity", + "HarmfulContentEntityType", + "IntellectualPropertyEntityType", + "PIIDetectionEntity", + "PIIDetectionEntityType", + "GuardrailExecutionStage", + "GuardrailAction", + "LogAction", + "BlockAction", + "LoggingSeverityLevel", + "GuardrailBlockException", + "GuardrailExclude", + "GuardrailTargetAdapter", + "register_guardrail_adapter", ] diff --git a/packages/uipath-platform/src/uipath/platform/guardrails/_guardrails_service.py b/packages/uipath-platform/src/uipath/platform/guardrails/_guardrails_service.py index ebfbaf33d..b73d810e7 100644 --- a/packages/uipath-platform/src/uipath/platform/guardrails/_guardrails_service.py +++ b/packages/uipath-platform/src/uipath/platform/guardrails/_guardrails_service.py @@ -1,4 +1,5 @@ -from typing import Any +import re +from typing import Any, Optional from httpx import HTTPStatusError from uipath.core.guardrails import ( @@ -7,12 +8,23 @@ ) from uipath.core.tracing import traced +from uipath.platform.constants import HEADER_GUARDRAILS_SOURCE + +from ..chat.llm_trace_context import build_trace_context_headers from ..common._base_service import BaseService from ..common._config import UiPathApiConfig from ..common._execution_context import UiPathExecutionContext +from ..common._job_context import header_job_key from ..common._models import Endpoint, RequestSpec from ..errors import EnrichedException -from .guardrails import BuiltInValidatorGuardrail +from .guardrails import BYO_VALIDATOR_TYPE, BuiltInValidatorGuardrail + +# x-uipath-traceparent-id header format: {version}-{trace_id}-{span_id}[-{trace_flags}] +# Based on W3C traceparent but allows 16- or 32-hex span IDs. +_TRACEPARENT_PATTERN = re.compile( + r"^[0-9a-f]{2}-[0-9a-f]{32}-(?P[0-9a-f]{16}|[0-9a-f]{32})(?:-[0-9a-f]{2})?$", + re.IGNORECASE, +) class GuardrailsService(BaseService): @@ -34,6 +46,31 @@ def __init__( ) -> None: super().__init__(config=config, execution_context=execution_context) + @staticmethod + def _extract_span_id_from_traceparent( + traceparent: Optional[str], + ) -> Optional[str]: + """Extract span ID from x-uipath-traceparent-id header and format as GUID. + + Args: + traceparent: Value from the ``x-uipath-traceparent-id`` response header. + Accepts 3-part ``"00-{trace_id}-{span_id}"`` or 4-part + ``"00-{trace_id}-{span_id}-{trace_flags}"``. Span ID may be + 16 or 32 hex chars. + + Returns: + Span ID formatted as lowercase GUID (8-4-4-4-12), or None if not parseable. + """ + if not traceparent: + return None + match = _TRACEPARENT_PATTERN.match(traceparent) + if not match: + return None + span_id_hex = match.group("span_id").lower() + # Pad to 32 chars for GUID conversion (span IDs may be 16 hex chars) + padded = span_id_hex.zfill(32) + return f"{padded[:8]}-{padded[8:12]}-{padded[12:16]}-{padded[16:20]}-{padded[20:32]}" + @staticmethod def _parse_result(result_str: str) -> GuardrailValidationResultType: """Parse result string from API response to GuardrailValidationResultType. @@ -78,22 +115,48 @@ def evaluate_guardrail( parameters = [ param.model_dump(by_alias=True) for param in guardrail.validator_parameters ] - payload = { + payload: dict[str, Any] = { "validator": guardrail.validator_type, "input": input_data if isinstance(input_data, str) else str(input_data), "parameters": parameters, + "guardrailName": guardrail.name, } + if guardrail.validator_type == BYO_VALIDATOR_TYPE: + if not guardrail.byo_validator_name: + raise ValueError( + "BYO (Bring Your Own) guardrails require byo_validator_name." + ) + payload["byoValidatorName"] = guardrail.byo_validator_name spec = RequestSpec( method="POST", endpoint=Endpoint("/agentsruntime_/api/execution/guardrails/validate"), json=payload, ) + # Include trace context headers for server-side span correlation, plus + # the execution source (x-uipath-guardrails-source) and job key headers + # for licensing/metering correlation. The execution source is read from + # the execution context, propagated from the runtime context. + trace_headers = build_trace_context_headers(extra_baggage=["source=agents"]) + source_headers: dict[str, str] = {} + execution_source = self._execution_context.execution_source + if execution_source: + source_headers[HEADER_GUARDRAILS_SOURCE] = execution_source + request_headers = { + **(spec.headers or {}), + **trace_headers, + **source_headers, + **header_job_key(), + } + span_id = None try: response = self.request( spec.method, url=spec.endpoint, json=spec.json, - headers=spec.headers, + headers=request_headers, + ) + span_id = self._extract_span_id_from_traceparent( + response.headers.get("x-uipath-traceparent-id") ) response_data = response.json() except EnrichedException as e: @@ -107,6 +170,11 @@ def evaluate_guardrail( and original_error.response ): try: + span_id = self._extract_span_id_from_traceparent( + original_error.response.headers.get( + "x-uipath-traceparent-id" + ) + ) response_data = original_error.response.json() except Exception: # If JSON parsing fails, re-raise the original exception @@ -127,9 +195,11 @@ def evaluate_guardrail( reason = response_data.get("details", "") # Prepare model data - model_data = { + model_data: dict[str, Any] = { "result": result.value, "reason": reason, } + if span_id: + model_data["spanId"] = span_id return GuardrailValidationResult.model_validate(model_data) diff --git a/packages/uipath-platform/src/uipath/platform/guardrails/decorators/__init__.py b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/__init__.py new file mode 100644 index 000000000..3db6a9c4f --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/__init__.py @@ -0,0 +1,70 @@ +"""Guardrail decorator framework for UiPath Platform. + +Provides the ``@guardrail`` decorator, built-in validators, actions, and an +adapter registry that framework integrations (e.g. *uipath-langchain*) use to +teach the decorator how to wrap their specific object types. +""" + +from ._actions import BlockAction, LogAction, LoggingSeverityLevel +from ._core import GuardrailExclude +from ._enums import ( + GuardrailExecutionStage, + HarmfulContentEntityType, + IntellectualPropertyEntityType, + PIIDetectionEntityType, +) +from ._exceptions import GuardrailBlockException +from ._guardrail import guardrail +from ._models import GuardrailAction, HarmfulContentEntity, PIIDetectionEntity +from ._registry import GuardrailTargetAdapter, register_guardrail_adapter +from .validators import ( + BuiltInGuardrailValidator, + ByoValidator, + CustomGuardrailValidator, + CustomValidator, + GuardrailValidatorBase, + HarmfulContentValidator, + IntellectualPropertyValidator, + LLMAsJudgeValidator, + PIIValidator, + PromptInjectionValidator, + RuleFunction, + UserPromptAttacksValidator, +) + +__all__ = [ + # Decorator + "guardrail", + # Validators + "GuardrailValidatorBase", + "BuiltInGuardrailValidator", + "ByoValidator", + "CustomGuardrailValidator", + "HarmfulContentValidator", + "IntellectualPropertyValidator", + "LLMAsJudgeValidator", + "PIIValidator", + "PromptInjectionValidator", + "UserPromptAttacksValidator", + "CustomValidator", + "RuleFunction", + # Models & enums + "HarmfulContentEntity", + "HarmfulContentEntityType", + "IntellectualPropertyEntityType", + "PIIDetectionEntity", + "PIIDetectionEntityType", + "GuardrailExecutionStage", + "GuardrailAction", + # Actions + "LogAction", + "BlockAction", + "LoggingSeverityLevel", + # Exception + "GuardrailBlockException", + # Exclude marker + "GuardrailExclude", + # Adapter registry + "GuardrailTargetAdapter", + "register_guardrail_adapter", +] diff --git a/packages/uipath-platform/src/uipath/platform/guardrails/decorators/_actions.py b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/_actions.py new file mode 100644 index 000000000..8e6489797 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/_actions.py @@ -0,0 +1,82 @@ +"""Built-in GuardrailAction implementations.""" + +import logging +from dataclasses import dataclass +from enum import Enum +from typing import Any, Optional + +from uipath.core.guardrails import ( + GuardrailValidationResult, + GuardrailValidationResultType, +) + +from ._exceptions import GuardrailBlockException +from ._models import GuardrailAction + + +class LoggingSeverityLevel(int, Enum): + """Logging severity level for :class:`LogAction`.""" + + ERROR = logging.ERROR + INFO = logging.INFO + WARNING = logging.WARNING + DEBUG = logging.DEBUG + + +@dataclass +class LogAction(GuardrailAction): + """Log guardrail violations without stopping execution. + + Args: + severity_level: Python logging level. Defaults to ``WARNING``. + message: Custom log message. If omitted, the validation reason is used. + """ + + severity_level: LoggingSeverityLevel = LoggingSeverityLevel.WARNING + message: Optional[str] = None + + def handle_validation_result( + self, + result: GuardrailValidationResult, + data: str | dict[str, Any], + guardrail_name: str, + ) -> str | dict[str, Any] | None: + """Log the violation and return ``None`` (no data modification).""" + if result.result == GuardrailValidationResultType.VALIDATION_FAILED: + msg = self.message or f"Failed: {result.reason}" + logging.getLogger(__name__).log( + self.severity_level, + "[GUARDRAIL] [%s] %s", + guardrail_name, + msg, + ) + return None + + +@dataclass +class BlockAction(GuardrailAction): + """Block execution by raising :class:`GuardrailBlockException`. + + Framework adapters catch ``GuardrailBlockException`` at the wrapper boundary + and convert it to their own runtime error type. + + Args: + title: Exception title. Defaults to a message derived from the guardrail name. + detail: Exception detail. Defaults to the validation reason. + """ + + title: Optional[str] = None + detail: Optional[str] = None + + def handle_validation_result( + self, + result: GuardrailValidationResult, + data: str | dict[str, Any], + guardrail_name: str, + ) -> str | dict[str, Any] | None: + """Raise :class:`GuardrailBlockException` when validation fails.""" + if result.result == GuardrailValidationResultType.VALIDATION_FAILED: + title = self.title or f"Guardrail [{guardrail_name}] blocked execution" + detail = self.detail or result.reason or "Guardrail validation failed" + raise GuardrailBlockException(title=title, detail=detail) + return None diff --git a/packages/uipath-platform/src/uipath/platform/guardrails/decorators/_core.py b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/_core.py new file mode 100644 index 000000000..ca168a1e0 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/_core.py @@ -0,0 +1,302 @@ +"""Core framework-agnostic utilities for guardrail decorators.""" + +import ast +import dataclasses +import inspect +import json +import logging +from typing import Annotated, Any, Callable, get_args, get_origin, get_type_hints + +from uipath.core.guardrails import ( + GuardrailValidationResult, +) + +from ._enums import GuardrailExecutionStage + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# GuardrailExclude marker +# --------------------------------------------------------------------------- + + +class GuardrailExclude: + """Marker to exclude a parameter from guardrail input serialization. + + Use with :data:`typing.Annotated` to prevent a specific function parameter + from being collected into the guardrail evaluation payload:: + + async def process( + text: str, + config: Annotated[dict, GuardrailExclude()], + ) -> str: ... + """ + + +# --------------------------------------------------------------------------- +# Evaluator type alias +# --------------------------------------------------------------------------- + +_EvaluatorFn = Callable[ + [ + "str | dict[str, Any]", # data + GuardrailExecutionStage, # stage + "dict[str, Any] | None", # input_data + "dict[str, Any] | None", # output_data + ], + GuardrailValidationResult, +] +"""Type alias for the unified evaluation callable used by all wrappers.""" + + +# --------------------------------------------------------------------------- +# Evaluator factory +# --------------------------------------------------------------------------- + + +def _make_evaluator( + validator: Any, + name: str, + description: str | None, + enabled_for_evals: bool, +) -> _EvaluatorFn: + """Return a unified evaluation callable. + + Delegates to ``validator.run()`` which each validator subclass implements + (:class:`BuiltInGuardrailValidator` hits the UiPath API; + :class:`CustomGuardrailValidator` runs a local Python rule). + + Args: + validator: :class:`GuardrailValidatorBase` instance. + name: Guardrail name — forwarded to ``validator.run()`` on each call. + description: Optional description — forwarded to ``validator.run()``. + enabled_for_evals: Whether active in evaluation scenarios. + + Returns: + Callable with signature ``(data, stage, input_data, output_data)``. + """ + + def _eval( + data: str | dict[str, Any], + stage: GuardrailExecutionStage, + input_data: dict[str, Any] | None, + output_data: dict[str, Any] | None, + ) -> GuardrailValidationResult: + return validator.run( + name, description, enabled_for_evals, data, stage, input_data, output_data + ) + + return _eval + + +# --------------------------------------------------------------------------- +# Parameter introspection +# --------------------------------------------------------------------------- + + +def _get_excluded_params(func: Any) -> set[str]: + """Return parameter names annotated with :class:`GuardrailExclude`. + + Args: + func: Callable to inspect. + + Returns: + Set of parameter names that should be excluded from guardrail input. + """ + try: + hints = get_type_hints(func, include_extras=True) + except Exception: + return set() + excluded: set[str] = set() + for name, hint in hints.items(): + if get_origin(hint) is Annotated: + for meta in get_args(hint)[1:]: + if isinstance(meta, GuardrailExclude): + excluded.add(name) + return excluded + + +# --------------------------------------------------------------------------- +# Serialization helpers +# --------------------------------------------------------------------------- + + +def _serialize_value(value: Any) -> Any: + """Serialize *value* to a JSON-compatible type for guardrail evaluation. + + Pydantic models → ``model_dump()``, dataclasses → ``asdict()``, + primitives → as-is, everything else → ``str()``. + """ + if value is None: + return None + if isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, dict): + return {k: _serialize_value(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_serialize_value(v) for v in value] + # Pydantic v2 + if hasattr(value, "model_dump"): + return value.model_dump() + # Pydantic v1 + if hasattr(value, "dict") and callable(value.dict): + try: + return value.dict() + except Exception: + pass + # dataclasses + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return dataclasses.asdict(value) + return str(value) + + +def _collect_input( + bound: inspect.BoundArguments, + excluded: set[str], +) -> dict[str, Any]: + """Collect non-excluded function parameters into a guardrail input dict. + + Args: + bound: Bound arguments from ``inspect.Signature.bind()``. + excluded: Parameter names to skip. + + Returns: + ``{param_name: serialized_value}`` for all non-excluded parameters. + """ + result: dict[str, Any] = {} + for name, value in bound.arguments.items(): + if name in excluded or name in ("self", "cls"): + continue + result[name] = _serialize_value(value) + return result + + +def _collect_output(return_value: Any) -> dict[str, Any]: + """Serialize a function return value into a dict for guardrail evaluation. + + Args: + return_value: The value returned by the wrapped function. + + Returns: + A ``dict`` representation suitable for guardrail evaluation. + """ + serialized = _serialize_value(return_value) + if isinstance(serialized, dict): + return serialized + return {"return": serialized} + + +def _reconstruct_output(original: Any, modified: Any) -> Any: + """Reconstruct a return value from a guardrail-modified payload. + + Args: + original: The original return value (used to determine target type). + modified: The modified value returned by the guardrail action. + + Returns: + Reconstructed value of the same type as *original* where possible. + """ + if modified is None: + return original + # Pydantic v2 model + dict modification → reconstruct via model_validate + if hasattr(original, "model_validate") and isinstance(modified, dict): + try: + return type(original).model_validate(modified) + except Exception: + pass + # Pydantic v1 + if hasattr(original, "parse_obj") and isinstance(modified, dict): + try: + return type(original).parse_obj(modified) + except Exception: + pass + return modified + + +def _apply_pre_modification( + bound: inspect.BoundArguments, + modified: Any, + excluded: set[str], +) -> None: + """Apply guardrail PRE-stage modifications back to bound function arguments. + + If the action returned a modified dict, keys matching non-excluded parameters + are updated in-place. If the action returned a plain string and there is exactly + one non-excluded parameter, that parameter is updated. + + Args: + bound: Bound arguments to mutate in-place. + modified: Value returned by the guardrail action. + excluded: Parameter names that were excluded from evaluation. + """ + if modified is None: + return + non_excluded = [ + n for n in bound.arguments if n not in excluded and n not in ("self", "cls") + ] + if isinstance(modified, dict): + for name in non_excluded: + if name in modified: + bound.arguments[name] = modified[name] + elif isinstance(modified, str) and len(non_excluded) == 1: + bound.arguments[non_excluded[0]] = modified + + +# --------------------------------------------------------------------------- +# Tool I/O normalisation helpers (used by LangChain adapter) +# --------------------------------------------------------------------------- + + +def _is_tool_call_envelope(tool_input: Any) -> bool: + """Return ``True`` if *tool_input* is a LangGraph tool-call envelope dict.""" + return ( + isinstance(tool_input, dict) + and "args" in tool_input + and tool_input.get("type") == "tool_call" + ) + + +def _extract_input(tool_input: Any) -> dict[str, Any]: + """Normalise tool input to a plain dict for rule / guardrail evaluation. + + LangGraph wraps tool inputs as ``{"name": ..., "args": {...}, "type": "tool_call"}``. + This function unwraps ``args`` so rules can access the actual tool arguments. + """ + if _is_tool_call_envelope(tool_input): + args = tool_input["args"] + if isinstance(args, dict): + return args + if isinstance(tool_input, dict): + return tool_input + return {"input": tool_input} + + +def _rewrap_input(original_tool_input: Any, modified_args: dict[str, Any]) -> Any: + """Re-wrap modified args back into the original tool-call envelope (if applicable).""" + if _is_tool_call_envelope(original_tool_input): + import copy + + wrapped = copy.copy(original_tool_input) + wrapped["args"] = modified_args + return wrapped + return modified_args + + +def _extract_output(result: Any) -> dict[str, Any]: + """Normalise tool output to a dict for guardrail / rule evaluation. + + Falls back to ``{"output": content}`` for plain strings and anything else. + """ + if isinstance(result, dict): + return result + if isinstance(result, str): + try: + parsed = json.loads(result) + return parsed if isinstance(parsed, dict) else {"output": parsed} + except ValueError: + try: + parsed = ast.literal_eval(result) + return parsed if isinstance(parsed, dict) else {"output": parsed} + except (ValueError, SyntaxError): + return {"output": result} + return {"output": result} diff --git a/packages/uipath-platform/src/uipath/platform/guardrails/decorators/_enums.py b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/_enums.py new file mode 100644 index 000000000..c88df5acd --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/_enums.py @@ -0,0 +1,93 @@ +"""Enums for guardrail decorators.""" + +from enum import Enum + + +class GuardrailExecutionStage(str, Enum): + """Execution stage for guardrails.""" + + PRE = "pre" + """Evaluate before the target executes.""" + + POST = "post" + """Evaluate after the target executes.""" + + PRE_AND_POST = "pre&post" + """Evaluate both before and after the target executes.""" + + +class PIIDetectionEntityType(str, Enum): + """PII detection entity types supported by UiPath guardrails. + + | Value | + |---| + | `PERSON` | + | `ADDRESS` | + | `DATE` | + | `PHONE_NUMBER` | + | `EUGPS_COORDINATES` | + | `EMAIL` | + | `CREDIT_CARD_NUMBER` | + | `INTERNATIONAL_BANKING_ACCOUNT_NUMBER` | + | `SWIFT_CODE` | + | `ABA_ROUTING_NUMBER` | + | `US_DRIVERS_LICENSE_NUMBER` | + | `UK_DRIVERS_LICENSE_NUMBER` | + | `US_INDIVIDUAL_TAXPAYER_IDENTIFICATION` | + | `UK_UNIQUE_TAXPAYER_NUMBER` | + | `US_BANK_ACCOUNT_NUMBER` | + | `US_SOCIAL_SECURITY_NUMBER` | + | `USUK_PASSPORT_NUMBER` | + | `URL` | + | `IP_ADDRESS` | + """ + + PERSON = "Person" + ADDRESS = "Address" + DATE = "Date" + PHONE_NUMBER = "PhoneNumber" + EUGPS_COORDINATES = "EugpsCoordinates" + EMAIL = "Email" + CREDIT_CARD_NUMBER = "CreditCardNumber" + INTERNATIONAL_BANKING_ACCOUNT_NUMBER = "InternationalBankingAccountNumber" + SWIFT_CODE = "SwiftCode" + ABA_ROUTING_NUMBER = "ABARoutingNumber" + US_DRIVERS_LICENSE_NUMBER = "USDriversLicenseNumber" + UK_DRIVERS_LICENSE_NUMBER = "UKDriversLicenseNumber" + US_INDIVIDUAL_TAXPAYER_IDENTIFICATION = "USIndividualTaxpayerIdentification" + UK_UNIQUE_TAXPAYER_NUMBER = "UKUniqueTaxpayerNumber" + US_BANK_ACCOUNT_NUMBER = "USBankAccountNumber" + US_SOCIAL_SECURITY_NUMBER = "USSocialSecurityNumber" + USUK_PASSPORT_NUMBER = "UsukPassportNumber" + URL = "URL" + IP_ADDRESS = "IPAddress" + + +class HarmfulContentEntityType(str, Enum): + """Harmful content entity types supported by UiPath guardrails. + + | Value | + |---| + | `HATE` | + | `SELF_HARM` | + | `SEXUAL` | + | `VIOLENCE` | + """ + + HATE = "Hate" + SELF_HARM = "SelfHarm" + SEXUAL = "Sexual" + VIOLENCE = "Violence" + + +class IntellectualPropertyEntityType(str, Enum): + """Intellectual property entity types supported by UiPath guardrails. + + | Value | + |---| + | `TEXT` | + | `CODE` | + """ + + TEXT = "Text" + CODE = "Code" diff --git a/packages/uipath-platform/src/uipath/platform/guardrails/decorators/_exceptions.py b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/_exceptions.py new file mode 100644 index 000000000..f4b7672e5 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/_exceptions.py @@ -0,0 +1,18 @@ +"""Exceptions for guardrail decorators.""" + + +class GuardrailBlockException(Exception): + """Raised by BlockAction when a guardrail blocks execution. + + Framework adapters (e.g. LangChain) should catch this and convert it to + their own runtime exception type at the outermost wrapper boundary. + + Args: + title: Brief title for the block event. + detail: Detailed reason for the block. + """ + + def __init__(self, title: str, detail: str) -> None: + self.title = title + self.detail = detail + super().__init__(f"{title}: {detail}") diff --git a/packages/uipath-platform/src/uipath/platform/guardrails/decorators/_guardrail.py b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/_guardrail.py new file mode 100644 index 000000000..d61b41c0d --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/_guardrail.py @@ -0,0 +1,224 @@ +"""Single ``@guardrail`` decorator for all guardrail types.""" + +import inspect +import logging +from functools import wraps +from typing import Any + +from ._core import ( + _apply_pre_modification, + _collect_input, + _collect_output, + _EvaluatorFn, + _get_excluded_params, + _make_evaluator, + _reconstruct_output, +) +from ._enums import GuardrailExecutionStage +from ._models import GuardrailAction +from ._registry import is_recognized_by_adapter, wrap_with_adapter +from .validators._base import GuardrailValidatorBase + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + + +def _run_pre( + evaluator: _EvaluatorFn, + action: GuardrailAction, + name: str, + bound: inspect.BoundArguments, + excluded: set[str], +) -> None: + """Evaluate PRE guardrail and apply any modifications to *bound* in-place.""" + input_data = _collect_input(bound, excluded) + try: + result = evaluator(input_data, GuardrailExecutionStage.PRE, input_data, None) + except Exception as exc: + logger.error("Error evaluating PRE guardrail %r: %s", name, exc, exc_info=True) + return + from uipath.core.guardrails import GuardrailValidationResultType + + if result.result == GuardrailValidationResultType.VALIDATION_FAILED: + modified = action.handle_validation_result(result, input_data, name) + _apply_pre_modification(bound, modified, excluded) + + +def _run_post( + evaluator: _EvaluatorFn, + action: GuardrailAction, + name: str, + bound: inspect.BoundArguments, + excluded: set[str], + return_value: Any, +) -> Any: + """Evaluate POST guardrail and return (possibly modified) return value.""" + input_data = _collect_input(bound, excluded) + output_data = _collect_output(return_value) + try: + result = evaluator( + output_data, GuardrailExecutionStage.POST, input_data, output_data + ) + except Exception as exc: + logger.error("Error evaluating POST guardrail %r: %s", name, exc, exc_info=True) + return return_value + from uipath.core.guardrails import GuardrailValidationResultType + + if result.result == GuardrailValidationResultType.VALIDATION_FAILED: + modified = action.handle_validation_result(result, output_data, name) + return _reconstruct_output(return_value, modified) + return return_value + + +def _wrap_function( + func: Any, + evaluator: _EvaluatorFn, + action: GuardrailAction, + name: str, + stage: GuardrailExecutionStage, + excluded: set[str], +) -> Any: + """Wrap *func* as a pure Python function with PRE/POST guardrail evaluation.""" + sig = inspect.signature(func) + + def _dispatch_return(return_value: Any) -> Any: + """For factory functions: if the return value is recognized by an adapter, wrap it.""" + if is_recognized_by_adapter(return_value): + return wrap_with_adapter(return_value, evaluator, action, name, stage) + return return_value + + if inspect.iscoroutinefunction(func): + + @wraps(func) + async def _wrapped_async(*args: Any, **kwargs: Any) -> Any: + bound = sig.bind(*args, **kwargs) + bound.apply_defaults() + if stage in ( + GuardrailExecutionStage.PRE, + GuardrailExecutionStage.PRE_AND_POST, + ): + _run_pre(evaluator, action, name, bound, excluded) + return_value = await func(*bound.args, **bound.kwargs) + return_value = _dispatch_return(return_value) + if stage in ( + GuardrailExecutionStage.POST, + GuardrailExecutionStage.PRE_AND_POST, + ): + # Only run POST on plain (non-adapter-wrapped) values + if not is_recognized_by_adapter(return_value): + return_value = _run_post( + evaluator, action, name, bound, excluded, return_value + ) + return return_value + + return _wrapped_async + + @wraps(func) + def _wrapped(*args: Any, **kwargs: Any) -> Any: + bound = sig.bind(*args, **kwargs) + bound.apply_defaults() + if stage in ( + GuardrailExecutionStage.PRE, + GuardrailExecutionStage.PRE_AND_POST, + ): + _run_pre(evaluator, action, name, bound, excluded) + return_value = func(*bound.args, **bound.kwargs) + return_value = _dispatch_return(return_value) + if stage in ( + GuardrailExecutionStage.POST, + GuardrailExecutionStage.PRE_AND_POST, + ): + if not is_recognized_by_adapter(return_value): + return_value = _run_post( + evaluator, action, name, bound, excluded, return_value + ) + return return_value + + return _wrapped + + +# --------------------------------------------------------------------------- +# Public @guardrail decorator +# --------------------------------------------------------------------------- + + +def guardrail( + func: Any = None, + *, + validator: GuardrailValidatorBase, + action: GuardrailAction, + name: str = "Guardrail", + description: str | None = None, + stage: GuardrailExecutionStage = GuardrailExecutionStage.PRE_AND_POST, + enabled_for_evals: bool = True, +) -> Any: + """Apply a guardrail to any callable — tool functions, LLM factories, agent nodes. + + When applied to a plain function or async function, the decorator collects + function parameters (PRE) and return value (POST) and evaluates them against + the guardrail. Use :class:`~._core.GuardrailExclude` to opt individual + parameters out of serialization. + + When applied to a factory function whose return value is recognised by a + registered framework adapter (e.g. a LangChain ``BaseChatModel``), the + returned object is wrapped so every subsequent ``invoke()`` call is guarded. + + Multiple ``@guardrail`` decorators can be stacked on the same callable. + + Args: + func: Callable to decorate. Supplied directly when used without parentheses. + validator: :class:`~.validators.GuardrailValidatorBase` defining what to check. + action: :class:`~._models.GuardrailAction` defining how to respond on violation. + name: Human-readable name for this guardrail instance. + description: Optional description passed to API-based guardrails. + stage: When to evaluate — ``PRE``, ``POST``, or ``PRE_AND_POST``. + Defaults to ``PRE_AND_POST``. + enabled_for_evals: Whether this guardrail is active in evaluation scenarios. + Defaults to ``True``. + + Returns: + The decorated callable (or framework object). + + Raises: + ValueError: If *action* is invalid, or the validator does not support + the requested stage. + GuardrailBlockException: Raised at runtime by :class:`~._actions.BlockAction` + when a violation is detected. + """ + if action is None: + raise ValueError("action must be provided") + if not isinstance(action, GuardrailAction): + raise ValueError("action must be an instance of GuardrailAction") + if not isinstance(enabled_for_evals, bool): + raise ValueError("enabled_for_evals must be a boolean") + + def _apply(obj: Any) -> Any: + # ------------------------------------------------------------------ + # 1. Adapter-recognised direct object (e.g. BaseTool after @tool) + # ------------------------------------------------------------------ + if is_recognized_by_adapter(obj): + validator.validate_stage(stage) + evaluator = _make_evaluator(validator, name, description, enabled_for_evals) + return wrap_with_adapter(obj, evaluator, action, name, stage) + + # ------------------------------------------------------------------ + # 2. Plain callable — wrap as pure function + # ------------------------------------------------------------------ + if callable(obj): + validator.validate_stage(stage) + evaluator = _make_evaluator(validator, name, description, enabled_for_evals) + excluded = _get_excluded_params(obj) + return _wrap_function(obj, evaluator, action, name, stage, excluded) + + raise ValueError( + f"@guardrail cannot be applied to {type(obj)!r}. " + "Target must be a callable or a framework-registered object. " + "Ensure the relevant framework adapter is imported before using @guardrail." + ) + + if func is None: + return _apply + return _apply(func) diff --git a/packages/uipath-platform/src/uipath/platform/guardrails/decorators/_models.py b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/_models.py new file mode 100644 index 000000000..8d86fbf39 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/_models.py @@ -0,0 +1,76 @@ +"""Models for guardrail decorators.""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Any + +from uipath.core.guardrails import GuardrailValidationResult + + +@dataclass +class PIIDetectionEntity: + """PII entity configuration with detection threshold. + + Args: + name: The entity type name (e.g. ``PIIDetectionEntityType.EMAIL``). + threshold: Confidence threshold (0.0 to 1.0) for detection. + """ + + name: str + threshold: float = 0.5 + + def __post_init__(self) -> None: + if not 0.0 <= self.threshold <= 1.0: + raise ValueError( + f"Threshold must be between 0.0 and 1.0, got {self.threshold}" + ) + + +@dataclass +class HarmfulContentEntity: + """Harmful content entity configuration with severity threshold. + + Args: + name: The entity type name (e.g. ``HarmfulContentEntityType.VIOLENCE``). + threshold: Severity threshold (0 to 6) for detection. Defaults to ``2``. + """ + + name: str + threshold: int = 2 + + def __post_init__(self) -> None: + if not 0 <= self.threshold <= 6: + raise ValueError(f"Threshold must be between 0 and 6, got {self.threshold}") + + +class GuardrailAction(ABC): + """Interface for defining custom actions when a guardrail violation is detected. + + Subclass this to implement custom behaviour on validation failure, such as + logging, blocking, or content sanitisation. Built-in implementations are + :class:`LogAction` and :class:`BlockAction`. + """ + + @abstractmethod + def handle_validation_result( + self, + result: GuardrailValidationResult, + data: str | dict[str, Any], + guardrail_name: str, + ) -> "str | dict[str, Any] | None": + """Handle a guardrail validation result. + + Called when guardrail validation fails. May return modified data to + sanitise/filter the validated content before execution continues, or + ``None`` to leave it unchanged. + + Args: + result: The validation result from the guardrails service. + data: The data that was validated (string or dictionary). Depending + on context this can be tool input, tool output, or message text. + guardrail_name: The name of the guardrail that triggered. + + Returns: + Modified data if the action wants to replace the original, or + ``None`` if no modification is needed. + """ diff --git a/packages/uipath-platform/src/uipath/platform/guardrails/decorators/_registry.py b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/_registry.py new file mode 100644 index 000000000..c4b7773b5 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/_registry.py @@ -0,0 +1,105 @@ +"""Adapter registry for guardrail target recognition and wrapping.""" + +from typing import Any, Protocol, runtime_checkable + +from ._core import _EvaluatorFn +from ._enums import GuardrailExecutionStage +from ._models import GuardrailAction + + +@runtime_checkable +class GuardrailTargetAdapter(Protocol): + """Protocol for framework-specific guardrail adapters. + + Implement this protocol to teach :func:`guardrail` how to handle objects + from a particular framework. Register instances via + :func:`register_guardrail_adapter`. + """ + + def recognize(self, target: Any) -> bool: + """Return ``True`` if this adapter handles *target*. + + Args: + target: Object being decorated or returned by a factory function. + + Returns: + ``True`` if this adapter can wrap *target*, ``False`` otherwise. + """ + ... + + def wrap( + self, + target: Any, + evaluator: _EvaluatorFn, + action: GuardrailAction, + name: str, + stage: GuardrailExecutionStage, + ) -> Any: + """Wrap *target* with guardrail enforcement logic. + + Args: + target: Object to wrap. + evaluator: Unified evaluation callable from :func:`_make_evaluator`. + action: Action to invoke on validation failure. + name: Human-readable guardrail name. + stage: When to evaluate (PRE, POST, or PRE_AND_POST). + + Returns: + Wrapped object, same type or duck-type compatible. + """ + ... + + +# Module-level registry. Later-registered adapters take priority (inserted at 0). +_adapters: list[GuardrailTargetAdapter] = [] + + +def register_guardrail_adapter(adapter: GuardrailTargetAdapter) -> None: + """Register a framework adapter for the ``@guardrail`` decorator. + + Later-registered adapters are tried first. + + Args: + adapter: An instance implementing :class:`GuardrailTargetAdapter`. + """ + _adapters.insert(0, adapter) + + +def is_recognized_by_adapter(target: Any) -> bool: + """Return ``True`` if any registered adapter recognizes *target*. + + Args: + target: The object being decorated. + + Returns: + ``True`` if a registered adapter handles *target*. + """ + for adapter in _adapters: + if adapter.recognize(target): + return True + return False + + +def wrap_with_adapter( + target: Any, + evaluator: _EvaluatorFn, + action: GuardrailAction, + name: str, + stage: GuardrailExecutionStage, +) -> Any: + """Ask the first matching adapter to wrap *target*. + + Args: + target: The object to wrap. + evaluator: Unified evaluation callable. + action: Action on violation. + name: Guardrail name. + stage: Execution stage. + + Returns: + Wrapped object, or *target* unchanged if no adapter handles it. + """ + for adapter in _adapters: + if adapter.recognize(target): + return adapter.wrap(target, evaluator, action, name, stage) + return target diff --git a/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/__init__.py b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/__init__.py new file mode 100644 index 000000000..b2ab5e6e2 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/__init__.py @@ -0,0 +1,30 @@ +"""Guardrail validators for the ``@guardrail`` decorator.""" + +from ._base import ( + BuiltInGuardrailValidator, + CustomGuardrailValidator, + GuardrailValidatorBase, +) +from .byo import ByoValidator +from .custom import CustomValidator, RuleFunction +from .harmful_content import HarmfulContentValidator +from .intellectual_property import IntellectualPropertyValidator +from .llm_as_judge import LLMAsJudgeValidator +from .pii import PIIValidator +from .prompt_injection import PromptInjectionValidator +from .user_prompt_attacks import UserPromptAttacksValidator + +__all__ = [ + "GuardrailValidatorBase", + "BuiltInGuardrailValidator", + "ByoValidator", + "CustomGuardrailValidator", + "HarmfulContentValidator", + "IntellectualPropertyValidator", + "LLMAsJudgeValidator", + "PIIValidator", + "PromptInjectionValidator", + "UserPromptAttacksValidator", + "CustomValidator", + "RuleFunction", +] diff --git a/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/_base.py b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/_base.py new file mode 100644 index 000000000..a9eaf5afd --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/_base.py @@ -0,0 +1,182 @@ +"""Abstract base classes for guardrail validators.""" + +from abc import ABC, abstractmethod +from typing import Any, ClassVar + +from uipath.core.guardrails import GuardrailValidationResult + +from uipath.platform.guardrails.guardrails import BuiltInValidatorGuardrail + +from .._enums import GuardrailExecutionStage + + +class GuardrailValidatorBase: + """Root base class for guardrail validators. + + Concrete validators should subclass either + :class:`BuiltInGuardrailValidator` (for UiPath API-backed validation) + or :class:`CustomGuardrailValidator` (for in-process Python validation). + """ + + supported_stages: ClassVar[list[GuardrailExecutionStage]] = [] + """Stages this validator supports. Empty list means all stages are allowed.""" + + def validate_stage(self, stage: GuardrailExecutionStage) -> None: + """Raise ``ValueError`` if *stage* is not in :attr:`supported_stages`. + + Args: + stage: Requested execution stage. + + Raises: + ValueError: If :attr:`supported_stages` is non-empty and *stage* is absent. + """ + if self.supported_stages and stage not in self.supported_stages: + raise ValueError( + f"{type(self).__name__} does not support stage {stage!r}. " + f"Supported stages: {[s.value for s in self.supported_stages]}" + ) + + def run( + self, + name: str, + description: str | None, + enabled_for_evals: bool, + data: "str | dict[str, Any]", + stage: GuardrailExecutionStage, + input_data: "dict[str, Any] | None", + output_data: "dict[str, Any] | None", + ) -> GuardrailValidationResult: + """Execute the guardrail evaluation. + + Called by the ``@guardrail`` decorator at each function invocation. + Subclasses override this via :class:`BuiltInGuardrailValidator` or + :class:`CustomGuardrailValidator`. + + Raises: + NotImplementedError: Always — subclass one of the two ABCs instead. + """ + raise NotImplementedError( + f"{type(self).__name__} must subclass BuiltInGuardrailValidator " + "or CustomGuardrailValidator and implement the required abstract method." + ) + + +class BuiltInGuardrailValidator(GuardrailValidatorBase, ABC): + """Base for validators that delegate to the UiPath Guardrails API. + + Subclass this and implement :meth:`get_built_in_guardrail` to create an + API-backed guardrail validator (e.g. PII detection, prompt injection). + + Example:: + + class MyValidator(BuiltInGuardrailValidator): + def get_built_in_guardrail(self, name, description, enabled_for_evals): + return BuiltInValidatorGuardrail( + id=str(uuid4()), + name=name, + ... + ) + """ + + @abstractmethod + def get_built_in_guardrail( + self, + name: str, + description: str | None, + enabled_for_evals: bool, + ) -> BuiltInValidatorGuardrail: + """Build the UiPath API guardrail definition for this validator. + + Args: + name: Name for the guardrail instance. + description: Optional description. + enabled_for_evals: Whether active in evaluation scenarios. + + Returns: + :class:`BuiltInValidatorGuardrail` ready to be sent to the API. + """ + ... + + def run( + self, + name: str, + description: str | None, + enabled_for_evals: bool, + data: "str | dict[str, Any]", + stage: GuardrailExecutionStage, + input_data: "dict[str, Any] | None", + output_data: "dict[str, Any] | None", + ) -> GuardrailValidationResult: + """Evaluate via the UiPath Guardrails API. + + Lazily initialises the ``UiPath`` client on the first call and reuses + it for all subsequent invocations. + """ + built_in = self.get_built_in_guardrail(name, description, enabled_for_evals) + if not hasattr(self, "_uipath"): + from uipath.platform import UiPath + + self._uipath: Any = UiPath() + return self._uipath.guardrails.evaluate_guardrail(data, built_in) + + +class CustomGuardrailValidator(GuardrailValidatorBase, ABC): + """Base for validators that run entirely in-process. + + Subclass this and implement :meth:`evaluate` to create a local guardrail + validator that requires no UiPath API call. + + Example:: + + class ProfanityValidator(CustomGuardrailValidator): + BANNED = {"badword"} + + def evaluate(self, data, stage, input_data, output_data): + text = (input_data or {}).get("message", "") + if any(w in text.lower() for w in self.BANNED): + return GuardrailValidationResult( + result=GuardrailValidationResultType.VALIDATION_FAILED, + reason="Profanity detected", + ) + return GuardrailValidationResult(result=GuardrailValidationResultType.PASSED) + """ + + @abstractmethod + def evaluate( + self, + data: "str | dict[str, Any]", + stage: GuardrailExecutionStage, + input_data: "dict[str, Any] | None", + output_data: "dict[str, Any] | None", + ) -> GuardrailValidationResult: + """Perform local validation without a UiPath API call. + + Return a result with ``VALIDATION_FAILED`` to **trigger** the guardrail + (causing the configured :class:`~uipath.platform.guardrails.decorators.GuardrailAction` + to fire), or ``PASSED`` to let execution continue unchanged. + + Args: + data: Primary data being evaluated. + stage: Current execution stage (PRE or POST). + input_data: Normalised function input dict, or ``None``. + output_data: Normalised function output dict, or ``None`` at PRE stage. + + Returns: + :class:`~uipath.core.guardrails.GuardrailValidationResult` — + return ``VALIDATION_FAILED`` to activate the guardrail, + ``PASSED`` to allow execution to continue. + """ + ... + + def run( + self, + name: str, + description: str | None, + enabled_for_evals: bool, + data: "str | dict[str, Any]", + stage: GuardrailExecutionStage, + input_data: "dict[str, Any] | None", + output_data: "dict[str, Any] | None", + ) -> GuardrailValidationResult: + """Delegate to :meth:`evaluate`.""" + return self.evaluate(data, stage, input_data, output_data) diff --git a/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/byo.py b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/byo.py new file mode 100644 index 000000000..425a84654 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/byo.py @@ -0,0 +1,94 @@ +"""Bring Your Own Guardrail (BYOG) validator.""" + +from typing import Sequence +from uuid import uuid4 + +from uipath.platform.guardrails.guardrails import ( + BYO_VALIDATOR_TYPE, + BuiltInValidatorGuardrail, + ValidatorParameter, +) + +from ._base import BuiltInGuardrailValidator + + +class ByoValidator(BuiltInGuardrailValidator): + """Validate data through a Bring Your Own Guardrail (BYOG) configuration. + + BYOG lets an organization plug its own safety validator (e.g. a customer + Azure Content Safety subscription, a vendor connector, or a custom + Integration Service connector) into UiPath guardrails. An admin first + creates the configuration under ``Admin -> AI Trust Layer -> Guardrails + Configurations``; this validator references it purely by its validator + name, which is unique per tenant. The Integration Service connection to + use is resolved server-side from the configuration, so an admin rebind is + always honored. + + Supported at all stages — BYO validator capabilities are connector-defined + and cannot be known statically, so no stage restriction is applied here. + + Example:: + + from uipath.platform.guardrails.decorators import ( + BlockAction, + ByoValidator, + guardrail, + ) + + byog_harmful_content = ByoValidator("my-harmful-content-guardrail") + + @guardrail(validator=byog_harmful_content, action=BlockAction()) + def summarize(text: str) -> str: + ... + + Args: + validator_name: The BYOG configuration's validator name + (``byoValidatorName``), as shown in Admin -> AI Trust Layer -> + Guardrails Configurations. Unique per tenant. + parameters: Optional list of validator parameters. BYO parameter + schemas are connector-defined, so values are passed through as-is. + + Raises: + ValueError: If *validator_name* is empty or whitespace. + """ + + def __init__( + self, + validator_name: str, + *, + parameters: Sequence[ValidatorParameter] | None = None, + ) -> None: + """Initialize ByoValidator with a BYOG configuration reference.""" + if not validator_name or not validator_name.strip(): + raise ValueError("validator_name must be a non-empty string") + self.validator_name = validator_name + self.parameters = list(parameters or []) + + def get_built_in_guardrail( + self, + name: str, + description: str | None, + enabled_for_evals: bool, + ) -> BuiltInValidatorGuardrail: + """Build a BYOG :class:`BuiltInValidatorGuardrail`. + + Args: + name: Name for the guardrail. + description: Optional description. + enabled_for_evals: Whether active in evaluation scenarios. + + Returns: + Configured :class:`BuiltInValidatorGuardrail` referencing the BYOG + configuration via ``byoValidatorName``. + """ + return BuiltInValidatorGuardrail( + id=str(uuid4()), + name=name, + description=description + or f"Bring Your Own Guardrail validation '{self.validator_name}'", + enabled_for_evals=enabled_for_evals, + guardrail_type="builtInValidator", + validator_type=BYO_VALIDATOR_TYPE, + validator_parameters=self.parameters, + byo_validator_name=self.validator_name, + ) diff --git a/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/custom.py b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/custom.py new file mode 100644 index 000000000..df6549600 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/custom.py @@ -0,0 +1,125 @@ +"""Custom (rule-based) guardrail validator.""" + +import inspect +from typing import Any, Callable + +from uipath.core.guardrails import ( + GuardrailValidationResult, + GuardrailValidationResultType, +) + +from .._enums import GuardrailExecutionStage +from ._base import CustomGuardrailValidator + +RuleFunction = ( + Callable[[dict[str, Any]], bool] | Callable[[dict[str, Any], dict[str, Any]], bool] +) +"""Type alias for custom rule functions passed to :class:`CustomValidator`. + +The rule must return ``True`` to **trigger** the guardrail (i.e. signal a +violation that causes the configured action to fire), or ``False`` to let +execution continue unchanged. + +It accepts either one parameter (the input or output dict) or two parameters +(input dict, output dict — POST stage only). + +Examples:: + + # Triggered when "donkey" appears in the joke argument + CustomValidator(lambda args: "donkey" in args.get("joke", "").lower()) + + # Triggered when the output joke exceeds 500 characters + CustomValidator(lambda args: len(args.get("joke", "")) > 500) + + # Two-parameter form: triggered at POST when output contains input keyword + CustomValidator(lambda inp, out: inp.get("topic", "") in out.get("joke", "")) +""" + + +class CustomValidator(CustomGuardrailValidator): + """Validate function input/output using a local Python rule function. + + No UiPath API call is made. Applicable at any stage. + + The *rule* is called with the collected parameter dict (PRE stage) or the + serialised return-value dict (POST stage). It must return ``True`` to + **activate** the guardrail — i.e. to signal a violation and invoke the + configured :class:`~uipath.platform.guardrails.decorators.GuardrailAction`. + Return ``False`` (or any falsy value) to let execution continue unchanged. + + Args: + rule: A :data:`RuleFunction` that returns ``True`` to trigger the + guardrail. Must accept 1 or 2 parameters. + + Raises: + ValueError: If *rule* is not callable or has an unsupported parameter count. + """ + + def __init__(self, rule: RuleFunction) -> None: + """Initialize CustomValidator with a rule callable.""" + if not callable(rule): + raise ValueError(f"rule must be callable, got {type(rule)}") + sig = inspect.signature(rule) + param_count = len(sig.parameters) + if param_count not in (1, 2): + raise ValueError(f"rule must have 1 or 2 parameters, got {param_count}") + self.rule = rule + self._param_count = param_count + + def evaluate( + self, + data: str | dict[str, Any], + stage: GuardrailExecutionStage, + input_data: dict[str, Any] | None, + output_data: dict[str, Any] | None, + ) -> GuardrailValidationResult: + """Run the rule against the collected input or output dict. + + The rule receives the PRE parameter dict or POST return-value dict and + must return ``True`` to **trigger** the guardrail (VALIDATION_FAILED), + or ``False`` to pass. + + Args: + data: Unused; the rule operates on *input_data* or *output_data*. + stage: Current stage (PRE or POST). + input_data: Collected function input dict. + output_data: Collected function output dict, or ``None`` at PRE stage. + + Returns: + :class:`~uipath.core.guardrails.GuardrailValidationResult` — + ``VALIDATION_FAILED`` when the rule returns ``True`` (guardrail + triggered), ``PASSED`` otherwise. + """ + try: + if self._param_count == 2: + if input_data is None or output_data is None: + return GuardrailValidationResult( + result=GuardrailValidationResultType.PASSED, + reason="Two-parameter rule skipped: input or output data unavailable", + ) + violation = self.rule(input_data, output_data) # type: ignore[call-arg] + else: + target = ( + input_data if stage == GuardrailExecutionStage.PRE else output_data + ) + if target is None: + return GuardrailValidationResult( + result=GuardrailValidationResultType.PASSED, + reason="Rule skipped: data unavailable at this stage", + ) + violation = self.rule(target) # type: ignore[call-arg] + except Exception as exc: + return GuardrailValidationResult( + result=GuardrailValidationResultType.PASSED, + reason=f"Rule raised exception: {exc}", + ) + + if violation: + return GuardrailValidationResult( + result=GuardrailValidationResultType.VALIDATION_FAILED, + reason="Rule detected violation", + ) + return GuardrailValidationResult( + result=GuardrailValidationResultType.PASSED, + reason="Rule passed", + ) diff --git a/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/harmful_content.py b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/harmful_content.py new file mode 100644 index 000000000..d186341d7 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/harmful_content.py @@ -0,0 +1,77 @@ +"""Harmful content detection guardrail validator.""" + +from typing import Any, Sequence +from uuid import uuid4 + +from uipath.platform.guardrails.guardrails import ( + BuiltInValidatorGuardrail, + EnumListParameterValue, + MapEnumParameterValue, +) + +from .._models import HarmfulContentEntity +from ._base import BuiltInGuardrailValidator + + +class HarmfulContentValidator(BuiltInGuardrailValidator): + """Validate data for harmful content using the UiPath API. + + Supported at all stages (PRE, POST, PRE_AND_POST). + + Args: + entities: One or more :class:`~uipath.platform.guardrails.decorators.HarmfulContentEntity` + instances specifying which harmful content categories to detect + and their severity thresholds. + + Raises: + ValueError: If *entities* is empty. + """ + + def __init__(self, entities: Sequence[HarmfulContentEntity]) -> None: + """Initialize HarmfulContentValidator with entities to detect.""" + if not entities: + raise ValueError("entities must be provided and non-empty") + self.entities = list(entities) + + def get_built_in_guardrail( + self, + name: str, + description: str | None, + enabled_for_evals: bool, + ) -> BuiltInValidatorGuardrail: + """Build a harmful content :class:`BuiltInValidatorGuardrail`. + + Args: + name: Name for the guardrail. + description: Optional description. + enabled_for_evals: Whether active in evaluation scenarios. + + Returns: + Configured :class:`BuiltInValidatorGuardrail` for harmful content detection. + """ + entity_names = [entity.name for entity in self.entities] + entity_thresholds: dict[str, Any] = { + entity.name: entity.threshold for entity in self.entities + } + + return BuiltInValidatorGuardrail( + id=str(uuid4()), + name=name, + description=description + or f"Detects harmful content: {', '.join(entity_names)}", + enabled_for_evals=enabled_for_evals, + guardrail_type="builtInValidator", + validator_type="harmful_content", + validator_parameters=[ + EnumListParameterValue( + parameter_type="enum-list", + id="harmfulContentEntities", + value=entity_names, + ), + MapEnumParameterValue( + parameter_type="map-enum", + id="harmfulContentEntityThresholds", + value=entity_thresholds, + ), + ], + ) diff --git a/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/intellectual_property.py b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/intellectual_property.py new file mode 100644 index 000000000..8a18e6a37 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/intellectual_property.py @@ -0,0 +1,67 @@ +"""Intellectual property detection guardrail validator.""" + +from typing import Sequence +from uuid import uuid4 + +from uipath.platform.guardrails.guardrails import ( + BuiltInValidatorGuardrail, + EnumListParameterValue, +) + +from .._enums import GuardrailExecutionStage +from ._base import BuiltInGuardrailValidator + + +class IntellectualPropertyValidator(BuiltInGuardrailValidator): + """Validate output for intellectual property violations using the UiPath API. + + Restricted to POST stage only — IP detection is an output-only concern. + + Args: + entities: One or more entity type strings (e.g. + ``IntellectualPropertyEntityType.TEXT``). + + Raises: + ValueError: If *entities* is empty. + """ + + supported_stages = [GuardrailExecutionStage.POST] + + def __init__(self, entities: Sequence[str]) -> None: + """Initialize IntellectualPropertyValidator with entities to detect.""" + if not entities: + raise ValueError("entities must be provided and non-empty") + self.entities = list(entities) + + def get_built_in_guardrail( + self, + name: str, + description: str | None, + enabled_for_evals: bool, + ) -> BuiltInValidatorGuardrail: + """Build an intellectual property :class:`BuiltInValidatorGuardrail`. + + Args: + name: Name for the guardrail. + description: Optional description. + enabled_for_evals: Whether active in evaluation scenarios. + + Returns: + Configured :class:`BuiltInValidatorGuardrail` for IP detection. + """ + return BuiltInValidatorGuardrail( + id=str(uuid4()), + name=name, + description=description + or f"Detects intellectual property: {', '.join(self.entities)}", + enabled_for_evals=enabled_for_evals, + guardrail_type="builtInValidator", + validator_type="intellectual_property", + validator_parameters=[ + EnumListParameterValue( + parameter_type="enum-list", + id="ipEntities", + value=self.entities, + ), + ], + ) diff --git a/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/llm_as_judge.py b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/llm_as_judge.py new file mode 100644 index 000000000..f639a6ef3 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/llm_as_judge.py @@ -0,0 +1,157 @@ +"""LLM-as-judge guardrail validator.""" + +from typing import Any, Sequence +from uuid import uuid4 + +from uipath.platform.guardrails.guardrails import ( + BuiltInValidatorGuardrail, + EnumParameterValue, + NumberParameterValue, + TextListParameterValue, + TextParameterValue, +) + +from ._base import BuiltInGuardrailValidator + +# Threshold scale matches the backend OOTB catalog: any float in [0, 6], default 2 +# (the catalog's UI step of 2 is only a slider increment; the server accepts any +# value in range). HIGHER = more lenient (only flag clear violations), LOWER = stricter. +_THRESHOLD_MIN = 0.0 +_THRESHOLD_MAX = 6.0 +_THRESHOLD_DEFAULT = 2.0 + +# Input limits mirror the backend OOTB catalog / LlmAsJudgeProviderApi; keep in sync. +_MAX_GUARDRAIL_TEXT_LENGTH = 4000 +_MAX_EXAMPLE_LENGTH = 1000 +_MAX_EXAMPLES_PER_LIST = 2 + + +class LLMAsJudgeValidator(BuiltInGuardrailValidator): + """Validate content against a natural-language rule via an LLM judge. + + The customer expresses a rule in natural language (``guardrail_text``) and picks a + judge model; a judge LLM decides whether the evaluated payload complies. Works at + any scope/stage the ``@guardrail`` decorator wraps (scope is implicit in the + decorated target; ``supported_stages`` is left as the default empty list, which + means all stages are allowed — both PRE and POST). + + Args: + guardrail_text: The natural-language rule the judge evaluates against + (at most 4000 characters). + model: The judge model to use (a model id supported by LLM Gateway). + positive_examples: Optional example payloads that comply with the rule + (at most 2 entries, each at most 1000 characters). + negative_examples: Optional example payloads that violate the rule + (at most 2 entries, each at most 1000 characters). + threshold: Strictness on a 0-6 scale (default 2); higher is more lenient. + + Raises: + ValueError: If ``guardrail_text``/``model`` are empty, ``threshold`` is + outside [0, 6], ``guardrail_text`` exceeds 4000 characters, either example + list has more than 2 entries, or any example exceeds 1000 characters. + """ + + def __init__( + self, + guardrail_text: str, + model: str, + *, + positive_examples: Sequence[str] | None = None, + negative_examples: Sequence[str] | None = None, + threshold: float = _THRESHOLD_DEFAULT, + ) -> None: + """Initialize LLMAsJudgeValidator with the rule, judge model, and options.""" + if not guardrail_text or not guardrail_text.strip(): + raise ValueError("guardrail_text must be a non-empty string") + if not model or not model.strip(): + raise ValueError("model must be a non-empty string") + if not _THRESHOLD_MIN <= threshold <= _THRESHOLD_MAX: + raise ValueError( + f"threshold must be between {_THRESHOLD_MIN} and {_THRESHOLD_MAX}, " + f"got {threshold}" + ) + if len(guardrail_text) > _MAX_GUARDRAIL_TEXT_LENGTH: + raise ValueError( + f"guardrail_text exceeds the {_MAX_GUARDRAIL_TEXT_LENGTH}-character " + f"limit (got {len(guardrail_text)})" + ) + positive_examples = list(positive_examples or []) + negative_examples = list(negative_examples or []) + for label, examples in ( + ("positive_examples", positive_examples), + ("negative_examples", negative_examples), + ): + if len(examples) > _MAX_EXAMPLES_PER_LIST: + raise ValueError( + f"{label} allows at most {_MAX_EXAMPLES_PER_LIST} examples " + f"(got {len(examples)})" + ) + if any(len(e) > _MAX_EXAMPLE_LENGTH for e in examples): + raise ValueError( + f"each {label} entry must be at most {_MAX_EXAMPLE_LENGTH} characters" + ) + self.guardrail_text = guardrail_text + self.model = model + self.positive_examples = positive_examples + self.negative_examples = negative_examples + self.threshold = threshold + + def get_built_in_guardrail( + self, + name: str, + description: str | None, + enabled_for_evals: bool, + ) -> BuiltInValidatorGuardrail: + """Build an LLM-as-judge :class:`BuiltInValidatorGuardrail`. + + Args: + name: Name for the guardrail. + description: Optional description. + enabled_for_evals: Whether active in evaluation scenarios. + + Returns: + Configured :class:`BuiltInValidatorGuardrail` for llm_as_judge. + """ + validator_parameters: list[Any] = [ + TextParameterValue( + parameter_type="text", + id="guardrailText", + value=self.guardrail_text, + ), + EnumParameterValue( + parameter_type="enum", + id="model", + value=self.model, + ), + NumberParameterValue( + parameter_type="number", + id="threshold", + value=self.threshold, + ), + ] + if self.positive_examples: + validator_parameters.append( + TextListParameterValue( + parameter_type="text-list", + id="positiveExamples", + value=self.positive_examples, + ) + ) + if self.negative_examples: + validator_parameters.append( + TextListParameterValue( + parameter_type="text-list", + id="negativeExamples", + value=self.negative_examples, + ) + ) + + return BuiltInValidatorGuardrail( + id=str(uuid4()), + name=name, + description=description or "LLM-as-judge evaluation", + enabled_for_evals=enabled_for_evals, + guardrail_type="builtInValidator", + validator_type="llm_as_judge", + validator_parameters=validator_parameters, + ) diff --git a/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/pii.py b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/pii.py new file mode 100644 index 000000000..64d0a47aa --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/pii.py @@ -0,0 +1,76 @@ +"""PII detection guardrail validator.""" + +from typing import Any, Sequence +from uuid import uuid4 + +from uipath.platform.guardrails.guardrails import ( + BuiltInValidatorGuardrail, + EnumListParameterValue, + MapEnumParameterValue, +) + +from .._models import PIIDetectionEntity +from ._base import BuiltInGuardrailValidator + + +class PIIValidator(BuiltInGuardrailValidator): + """Validate data for PII entities using the UiPath PII detection API. + + Supported at all stages. + + Args: + entities: One or more :class:`~uipath.platform.guardrails.decorators.PIIDetectionEntity` + instances specifying which PII types to detect and their confidence thresholds. + + Raises: + ValueError: If *entities* is empty. + """ + + def __init__(self, entities: Sequence[PIIDetectionEntity]) -> None: + """Initialize PIIValidator with a list of entities to detect.""" + if not entities: + raise ValueError("entities must be provided and non-empty") + self.entities = list(entities) + + def get_built_in_guardrail( + self, + name: str, + description: str | None, + enabled_for_evals: bool, + ) -> BuiltInValidatorGuardrail: + """Build a PII detection :class:`BuiltInValidatorGuardrail`. + + Args: + name: Name for the guardrail. + description: Optional description. + enabled_for_evals: Whether active in evaluation scenarios. + + Returns: + Configured :class:`BuiltInValidatorGuardrail` for PII detection. + """ + entity_names = [entity.name for entity in self.entities] + entity_thresholds: dict[str, Any] = { + entity.name: entity.threshold for entity in self.entities + } + + return BuiltInValidatorGuardrail( + id=str(uuid4()), + name=name, + description=description + or f"Detects PII entities: {', '.join(entity_names)}", + enabled_for_evals=enabled_for_evals, + guardrail_type="builtInValidator", + validator_type="pii_detection", + validator_parameters=[ + EnumListParameterValue( + parameter_type="enum-list", + id="entities", + value=entity_names, + ), + MapEnumParameterValue( + parameter_type="map-enum", + id="entityThresholds", + value=entity_thresholds, + ), + ], + ) diff --git a/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/prompt_injection.py b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/prompt_injection.py new file mode 100644 index 000000000..b0943b396 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/prompt_injection.py @@ -0,0 +1,65 @@ +"""Prompt injection detection guardrail validator.""" + +from uuid import uuid4 + +from uipath.platform.guardrails.guardrails import ( + BuiltInValidatorGuardrail, + NumberParameterValue, +) + +from .._enums import GuardrailExecutionStage +from ._base import BuiltInGuardrailValidator + + +class PromptInjectionValidator(BuiltInGuardrailValidator): + """Validate input for prompt injection attacks via the UiPath API. + + Restricted to PRE stage only — prompt injection is an input-only concern. + + Args: + threshold: Detection confidence threshold (0.0–1.0). Defaults to ``0.5``. + + Raises: + ValueError: If *threshold* is outside [0.0, 1.0]. + """ + + supported_stages = [GuardrailExecutionStage.PRE] + + def __init__(self, threshold: float = 0.5) -> None: + """Initialize PromptInjectionValidator with a detection threshold.""" + if not 0.0 <= threshold <= 1.0: + raise ValueError(f"threshold must be between 0.0 and 1.0, got {threshold}") + self.threshold = threshold + + def get_built_in_guardrail( + self, + name: str, + description: str | None, + enabled_for_evals: bool, + ) -> BuiltInValidatorGuardrail: + """Build a prompt injection :class:`BuiltInValidatorGuardrail`. + + Args: + name: Name for the guardrail. + description: Optional description. + enabled_for_evals: Whether active in evaluation scenarios. + + Returns: + Configured :class:`BuiltInValidatorGuardrail` for prompt injection. + """ + return BuiltInValidatorGuardrail( + id=str(uuid4()), + name=name, + description=description + or f"Detects prompt injection with threshold {self.threshold}", + enabled_for_evals=enabled_for_evals, + guardrail_type="builtInValidator", + validator_type="prompt_injection", + validator_parameters=[ + NumberParameterValue( + parameter_type="number", + id="threshold", + value=self.threshold, + ), + ], + ) diff --git a/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/user_prompt_attacks.py b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/user_prompt_attacks.py new file mode 100644 index 000000000..7275acc25 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/guardrails/decorators/validators/user_prompt_attacks.py @@ -0,0 +1,44 @@ +"""User prompt attacks detection guardrail validator.""" + +from uuid import uuid4 + +from uipath.platform.guardrails.guardrails import BuiltInValidatorGuardrail + +from .._enums import GuardrailExecutionStage +from ._base import BuiltInGuardrailValidator + + +class UserPromptAttacksValidator(BuiltInGuardrailValidator): + """Validate input for user prompt attacks via the UiPath API. + + Restricted to PRE stage only — prompt attacks are an input-only concern. + Takes no parameters. + """ + + supported_stages = [GuardrailExecutionStage.PRE] + + def get_built_in_guardrail( + self, + name: str, + description: str | None, + enabled_for_evals: bool, + ) -> BuiltInValidatorGuardrail: + """Build a user prompt attacks :class:`BuiltInValidatorGuardrail`. + + Args: + name: Name for the guardrail. + description: Optional description. + enabled_for_evals: Whether active in evaluation scenarios. + + Returns: + Configured :class:`BuiltInValidatorGuardrail` for user prompt attacks. + """ + return BuiltInValidatorGuardrail( + id=str(uuid4()), + name=name, + description=description or "Detects user prompt attacks", + enabled_for_evals=enabled_for_evals, + guardrail_type="builtInValidator", + validator_type="user_prompt_attacks", + validator_parameters=[], + ) diff --git a/packages/uipath-platform/src/uipath/platform/guardrails/guardrails.py b/packages/uipath-platform/src/uipath/platform/guardrails/guardrails.py index cfc1e295f..dace18019 100644 --- a/packages/uipath-platform/src/uipath/platform/guardrails/guardrails.py +++ b/packages/uipath-platform/src/uipath/platform/guardrails/guardrails.py @@ -37,12 +37,52 @@ class NumberParameterValue(BaseModel): model_config = ConfigDict(populate_by_name=True, extra="allow") +class EnumParameterValue(BaseModel): + """Single-select enum parameter value.""" + + parameter_type: Literal["enum"] = Field(alias="$parameterType") + id: str + value: str + + model_config = ConfigDict(populate_by_name=True, extra="allow") + + +class TextParameterValue(BaseModel): + """Free-text parameter value.""" + + parameter_type: Literal["text"] = Field(alias="$parameterType") + id: str + value: str + + model_config = ConfigDict(populate_by_name=True, extra="allow") + + +class TextListParameterValue(BaseModel): + """List-of-text parameter value.""" + + parameter_type: Literal["text-list"] = Field(alias="$parameterType") + id: str + value: list[str] + + model_config = ConfigDict(populate_by_name=True, extra="allow") + + ValidatorParameter = Annotated[ - EnumListParameterValue | MapEnumParameterValue | NumberParameterValue, + EnumListParameterValue + | MapEnumParameterValue + | NumberParameterValue + | EnumParameterValue + | TextParameterValue + | TextListParameterValue, Field(discriminator="parameter_type"), ] +#: Sentinel ``validator_type`` for Bring Your Own Guardrail (BYOG) guardrails; the +#: connector-backed configuration is referenced by ``byo_validator_name`` instead. +BYO_VALIDATOR_TYPE = "byo" + + class BuiltInValidatorGuardrail(BaseGuardrail): """Built-in validator guardrail model.""" @@ -51,6 +91,7 @@ class BuiltInValidatorGuardrail(BaseGuardrail): validator_parameters: list[ValidatorParameter] = Field( default_factory=list, alias="validatorParameters" ) + byo_validator_name: str | None = Field(default=None, alias="byoValidatorName") model_config = ConfigDict(populate_by_name=True, extra="allow") diff --git a/packages/uipath-platform/src/uipath/platform/memory/__init__.py b/packages/uipath-platform/src/uipath/platform/memory/__init__.py new file mode 100644 index 000000000..31e364814 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/memory/__init__.py @@ -0,0 +1,39 @@ +"""Init file for memory module.""" + +from ._memory_service import MemoryService +from .memory import ( + CachedRecall, + EscalationMemoryIngestRequest, + EscalationMemoryMatch, + EscalationMemorySearchResponse, + FieldSettings, + MemoryMatch, + MemoryMatchField, + MemorySearchRequest, + MemorySearchResponse, + MemorySpace, + MemorySpaceCreateRequest, + MemorySpaceListResponse, + SearchField, + SearchMode, + SearchSettings, +) + +__all__ = [ + "CachedRecall", + "EscalationMemoryIngestRequest", + "EscalationMemoryMatch", + "EscalationMemorySearchResponse", + "FieldSettings", + "MemoryMatch", + "MemoryMatchField", + "MemorySearchRequest", + "MemorySearchResponse", + "MemoryService", + "MemorySpace", + "MemorySpaceCreateRequest", + "MemorySpaceListResponse", + "SearchField", + "SearchMode", + "SearchSettings", +] diff --git a/packages/uipath-platform/src/uipath/platform/memory/_memory_service.py b/packages/uipath-platform/src/uipath/platform/memory/_memory_service.py new file mode 100644 index 000000000..73d788f80 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/memory/_memory_service.py @@ -0,0 +1,493 @@ +"""Memory Spaces service. + +Memory space CRUD (create/list) goes through ECS v2. +Search and escalation memory operations go through LLMOps, which +enriches traces/feedback before forwarding to ECS. +""" + +from typing import Any, Optional + +from uipath.core.tracing import traced + +from ..common._base_service import BaseService +from ..common._bindings import resource_override +from ..common._config import UiPathApiConfig +from ..common._execution_context import UiPathExecutionContext +from ..common._folder_context import FolderContext, header_folder +from ..common._models import Endpoint, RequestSpec +from ..orchestrator._folder_service import FolderService +from .memory import ( + EscalationMemoryIngestRequest, + EscalationMemorySearchResponse, + MemorySearchRequest, + MemorySearchResponse, + MemorySpace, + MemorySpaceCreateRequest, + MemorySpaceListResponse, +) + +_MEMORY_SPACES_BASE = "/ecs_/v2/episodicmemories" +_LLMOPS_AGENT_BASE = "/llmopstenant_/api/Agent/memory" + + +class MemoryService(FolderContext, BaseService): + """Service for Agent Memory Spaces. + + Agent Memory allows agents to persist context across jobs using dynamic + few-shot retrieval. Memory spaces are folder-scoped and managed via ECS. + Search is routed through LLMOps, which handles trace/feedback enrichment + and system prompt injection. Escalation memory enables agents to recall + previously resolved escalation outcomes. + """ + + def __init__( + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + folders_service: FolderService, + ) -> None: + super().__init__(config=config, execution_context=execution_context) + self._folders_service = folders_service + + # ── Memory space operations (ECS) ────────────────────────────────── + + @resource_override(resource_type="memorySpace") + @traced(name="memory_create", run_type="uipath") + def create( + self, + name: str, + description: Optional[str] = None, + is_encrypted: Optional[bool] = None, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> MemorySpace: + """Create a new memory space. + + Args: + name: The name of the memory space (max 128 chars). + description: Optional description (max 1024 chars). + is_encrypted: Whether the memory space should be encrypted. + folder_key: The folder key for the operation. + folder_path: The folder path for the operation. + + Returns: + MemorySpace: The created memory space. + """ + spec = self._create_spec( + name, description, is_encrypted, folder_key, folder_path + ) + response = self.request( + spec.method, + spec.endpoint, + json=spec.json, + headers=spec.headers, + ).json() + return MemorySpace.model_validate(response) + + @resource_override(resource_type="memorySpace") + @traced(name="memory_create", run_type="uipath") + async def create_async( + self, + name: str, + description: Optional[str] = None, + is_encrypted: Optional[bool] = None, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> MemorySpace: + """Asynchronously create a new memory space. + + Args: + name: The name of the memory space (max 128 chars). + description: Optional description (max 1024 chars). + is_encrypted: Whether the memory space should be encrypted. + folder_key: The folder key for the operation. + folder_path: The folder path for the operation. + + Returns: + MemorySpace: The created memory space. + """ + spec = self._create_spec( + name, description, is_encrypted, folder_key, folder_path + ) + response = ( + await self.request_async( + spec.method, + spec.endpoint, + json=spec.json, + headers=spec.headers, + ) + ).json() + return MemorySpace.model_validate(response) + + @traced(name="memory_list", run_type="uipath") + def list( + self, + filter: Optional[str] = None, + orderby: Optional[str] = None, + top: Optional[int] = None, + skip: Optional[int] = None, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> MemorySpaceListResponse: + """List memory spaces with optional OData query parameters. + + Args: + filter: OData $filter expression. + orderby: OData $orderby expression. + top: Maximum number of results. + skip: Number of results to skip. + folder_key: The folder key for the operation. + folder_path: The folder path for the operation. + + Returns: + MemorySpaceListResponse: The list of memory spaces. + """ + spec = self._list_spec(filter, orderby, top, skip, folder_key, folder_path) + response = self.request( + spec.method, + spec.endpoint, + params=spec.params, + headers=spec.headers, + ).json() + return MemorySpaceListResponse.model_validate(response) + + @traced(name="memory_list", run_type="uipath") + async def list_async( + self, + filter: Optional[str] = None, + orderby: Optional[str] = None, + top: Optional[int] = None, + skip: Optional[int] = None, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> MemorySpaceListResponse: + """Asynchronously list memory spaces. + + Args: + filter: OData $filter expression. + orderby: OData $orderby expression. + top: Maximum number of results. + skip: Number of results to skip. + folder_key: The folder key for the operation. + folder_path: The folder path for the operation. + + Returns: + MemorySpaceListResponse: The list of memory spaces. + """ + spec = self._list_spec(filter, orderby, top, skip, folder_key, folder_path) + response = ( + await self.request_async( + spec.method, + spec.endpoint, + params=spec.params, + headers=spec.headers, + ) + ).json() + return MemorySpaceListResponse.model_validate(response) + + # ── Search (LLMOps) ─────────────────────────────────────────────── + + @traced(name="memory_search", run_type="uipath") + def search( + self, + memory_space_id: str, + request: MemorySearchRequest, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> MemorySearchResponse: + """Search a memory space via LLMOps. + + Returns search results with scores and a systemPromptInjection + string ready for the agent loop. + + Args: + memory_space_id: The GUID of the memory space. + request: The search request payload. + folder_key: The folder key for the operation. + folder_path: The folder path for the operation. + + Returns: + MemorySearchResponse: Results, metadata, and system prompt injection. + """ + spec = self._search_spec(memory_space_id, folder_key, folder_path) + response = self.request( + spec.method, + spec.endpoint, + json=request.model_dump(by_alias=True, exclude_none=True), + headers=spec.headers, + ).json() + return MemorySearchResponse.model_validate(response) + + @traced(name="memory_search", run_type="uipath") + async def search_async( + self, + memory_space_id: str, + request: MemorySearchRequest, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> MemorySearchResponse: + """Asynchronously search a memory space via LLMOps. + + Returns search results with scores and a systemPromptInjection + string ready for the agent loop. + + Args: + memory_space_id: The GUID of the memory space. + request: The search request payload. + folder_key: The folder key for the operation. + folder_path: The folder path for the operation. + + Returns: + MemorySearchResponse: Results, metadata, and system prompt injection. + """ + spec = self._search_spec(memory_space_id, folder_key, folder_path) + response = ( + await self.request_async( + spec.method, + spec.endpoint, + json=request.model_dump(by_alias=True, exclude_none=True), + headers=spec.headers, + ) + ).json() + return MemorySearchResponse.model_validate(response) + + # ── Escalation memory (LLMOps) ──────────────────────────────────── + + @traced(name="memory_escalation_search", run_type="uipath") + def escalation_search( + self, + memory_space_id: str, + request: MemorySearchRequest, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> EscalationMemorySearchResponse: + """Search escalation memory for previously resolved outcomes. + + Allows agents to recall past escalation resolutions to avoid + re-escalating for similar situations. + + Args: + memory_space_id: The GUID of the memory space. + request: The search request payload (same as regular search). + folder_key: The folder key for the operation. + folder_path: The folder path for the operation. + + Returns: + EscalationMemorySearchResponse: Matched escalation outcomes. + """ + spec = self._escalation_search_spec(memory_space_id, folder_key, folder_path) + response = self.request( + spec.method, + spec.endpoint, + json=request.model_dump(by_alias=True, exclude_none=True), + headers=spec.headers, + ).json() + return EscalationMemorySearchResponse.model_validate(response) + + @traced(name="memory_escalation_search", run_type="uipath") + async def escalation_search_async( + self, + memory_space_id: str, + request: MemorySearchRequest, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> EscalationMemorySearchResponse: + """Asynchronously search escalation memory for previously resolved outcomes. + + Allows agents to recall past escalation resolutions to avoid + re-escalating for similar situations. + + Args: + memory_space_id: The GUID of the memory space. + request: The search request payload (same as regular search). + folder_key: The folder key for the operation. + folder_path: The folder path for the operation. + + Returns: + EscalationMemorySearchResponse: Matched escalation outcomes. + """ + spec = self._escalation_search_spec(memory_space_id, folder_key, folder_path) + response = ( + await self.request_async( + spec.method, + spec.endpoint, + json=request.model_dump(by_alias=True, exclude_none=True), + headers=spec.headers, + ) + ).json() + return EscalationMemorySearchResponse.model_validate(response) + + @traced(name="memory_escalation_ingest", run_type="uipath") + def escalation_ingest( + self, + memory_space_id: str, + request: EscalationMemoryIngestRequest, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> None: + """Ingest a resolved escalation outcome into memory. + + Persists the outcome so future agent runs can recall it + without re-escalating. + + Args: + memory_space_id: The GUID of the memory space. + request: The escalation ingest payload. + folder_key: The folder key for the operation. + folder_path: The folder path for the operation. + """ + spec = self._escalation_ingest_spec(memory_space_id, folder_key, folder_path) + self.request( + spec.method, + spec.endpoint, + json=request.model_dump(by_alias=True, exclude_none=True), + headers=spec.headers, + ) + + @traced(name="memory_escalation_ingest", run_type="uipath") + async def escalation_ingest_async( + self, + memory_space_id: str, + request: EscalationMemoryIngestRequest, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> None: + """Asynchronously ingest a resolved escalation outcome into memory. + + Persists the outcome so future agent runs can recall it + without re-escalating. + + Args: + memory_space_id: The GUID of the memory space. + request: The escalation ingest payload. + folder_key: The folder key for the operation. + folder_path: The folder path for the operation. + """ + spec = self._escalation_ingest_spec(memory_space_id, folder_key, folder_path) + await self.request_async( + spec.method, + spec.endpoint, + json=request.model_dump(by_alias=True, exclude_none=True), + headers=spec.headers, + ) + + # ── Private spec builders ───────────────────────────────────────── + + def _resolve_folder( + self, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> Optional[str]: + """Resolve the folder key, supporting folder_path lookup for serverless. + + Priority: + 1. Explicit folder_key argument + 2. Explicit folder_path argument → resolve via FolderService + 3. UIPATH_FOLDER_KEY env var (via FolderContext._folder_key) + 4. UIPATH_FOLDER_PATH env var → resolve via FolderService + """ + if folder_key is None and folder_path is not None: + folder_key = self._folders_service.retrieve_key(folder_path=folder_path) + + if folder_key is None and folder_path is None: + folder_key = self._folder_key or ( + self._folders_service.retrieve_key(folder_path=self._folder_path) + if self._folder_path + else None + ) + + return folder_key + + # -- ECS specs -- + + def _create_spec( + self, + name: str, + description: Optional[str], + is_encrypted: Optional[bool], + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> RequestSpec: + folder_key = self._resolve_folder(folder_key, folder_path) + body = MemorySpaceCreateRequest( + name=name, + description=description, + is_encrypted=is_encrypted, + ) + return RequestSpec( + method="POST", + endpoint=Endpoint(f"{_MEMORY_SPACES_BASE}/create"), + json=body.model_dump(by_alias=True, exclude_none=True), + headers={**header_folder(folder_key, None)}, + ) + + def _list_spec( + self, + filter: Optional[str], + orderby: Optional[str], + top: Optional[int], + skip: Optional[int], + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> RequestSpec: + folder_key = self._resolve_folder(folder_key, folder_path) + params: dict[str, Any] = {} + if filter is not None: + params["$filter"] = filter + if orderby is not None: + params["$orderby"] = orderby + if top is not None: + params["$top"] = top + if skip is not None: + params["$skip"] = skip + return RequestSpec( + method="GET", + endpoint=Endpoint(_MEMORY_SPACES_BASE), + params=params, + headers={**header_folder(folder_key, None)}, + ) + + # -- LLMOps specs -- + + def _search_spec( + self, + memory_space_id: str, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> RequestSpec: + folder_key = self._resolve_folder(folder_key, folder_path) + return RequestSpec( + method="POST", + endpoint=Endpoint(f"{_LLMOPS_AGENT_BASE}/{memory_space_id}/search"), + headers={**header_folder(folder_key, None)}, + ) + + def _escalation_search_spec( + self, + memory_space_id: str, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> RequestSpec: + folder_key = self._resolve_folder(folder_key, folder_path) + return RequestSpec( + method="POST", + endpoint=Endpoint( + f"{_LLMOPS_AGENT_BASE}/{memory_space_id}/escalation/search" + ), + headers={**header_folder(folder_key, None)}, + ) + + def _escalation_ingest_spec( + self, + memory_space_id: str, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> RequestSpec: + folder_key = self._resolve_folder(folder_key, folder_path) + return RequestSpec( + method="POST", + endpoint=Endpoint( + f"{_LLMOPS_AGENT_BASE}/{memory_space_id}/escalation/ingest" + ), + headers={**header_folder(folder_key, None)}, + ) diff --git a/packages/uipath-platform/src/uipath/platform/memory/memory.py b/packages/uipath-platform/src/uipath/platform/memory/memory.py new file mode 100644 index 000000000..aadffbc79 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/memory/memory.py @@ -0,0 +1,191 @@ +"""Pydantic models for the Memory Spaces API. + +Memory space CRUD goes through ECS v2. Search goes through LLMOps, +which enriches traces/feedback before forwarding to ECS. +Escalation memory operations also go through LLMOps. +""" + +from enum import Enum +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field + +# ── Enums ────────────────────────────────────────────────────────────── + + +class SearchMode(str, Enum): + """Search mode for memory space queries.""" + + Hybrid = "Hybrid" + Semantic = "Semantic" + + +# ── Shared field models (used by both ECS and LLMOps) ───────────────── + + +class FieldSettings(BaseModel): + """Per-field search settings (optional overrides).""" + + model_config = ConfigDict(populate_by_name=True) + + weight: float = Field(default=1.0, alias="weight", ge=0.0, le=1.0) + threshold: Optional[float] = Field(None, alias="threshold", ge=0.0, le=1.0) + search_mode: Optional[SearchMode] = Field(None, alias="searchMode") + + +class SearchField(BaseModel): + """A field in a search request, with per-field settings.""" + + model_config = ConfigDict(populate_by_name=True) + + key_path: List[str] = Field(..., alias="keyPath", min_length=1) + value: str = Field(..., alias="value", min_length=1) + settings: FieldSettings = Field(default_factory=FieldSettings, alias="settings") + + +class SearchSettings(BaseModel): + """Top-level search settings.""" + + model_config = ConfigDict(populate_by_name=True) + + threshold: float = Field(default=0.0, alias="threshold", ge=0.0, le=1.0) + result_count: int = Field(default=1, alias="resultCount", ge=1, le=10) + search_mode: SearchMode = Field(..., alias="searchMode") + + +class MemoryMatchField(BaseModel): + """A field within a search result, with scoring details.""" + + model_config = ConfigDict(populate_by_name=True) + + key_path: List[str] = Field(..., alias="keyPath") + value: str = Field(..., alias="value") + weight: float = Field(..., alias="weight") + score: float = Field(..., alias="score") + weighted_score: float = Field(..., alias="weightedScore") + + +# ── ECS request models (memory space CRUD) ──────────────────────────── + + +class MemorySpaceCreateRequest(BaseModel): + """Request payload for creating a memory space (ECS).""" + + model_config = ConfigDict(populate_by_name=True) + + name: str = Field(..., alias="name", max_length=128, min_length=1) + description: Optional[str] = Field(None, alias="description", max_length=1024) + is_encrypted: Optional[bool] = Field(None, alias="isEncrypted") + + +# ── ECS response models ─────────────────────────────────────────────── + + +class MemorySpace(BaseModel): + """A memory space (folder-scoped, from ECS).""" + + model_config = ConfigDict(populate_by_name=True) + + id: str = Field(..., alias="id") + name: str = Field(..., alias="name") + description: Optional[str] = Field(None, alias="description") + last_queried: Optional[str] = Field(None, alias="lastQueried") + memories_count: int = Field(default=0, alias="memoriesCount") + folder_key: str = Field(..., alias="folderKey") + created_by_user_id: Optional[str] = Field(None, alias="createdByUserId") + is_encrypted: bool = Field(default=False, alias="isEncrypted") + + +class MemorySpaceListResponse(BaseModel): + """OData response from listing memory spaces (ECS).""" + + model_config = ConfigDict(populate_by_name=True) + + value: List[MemorySpace] = Field(default_factory=list, alias="value") + + +# ── LLMOps search models ────────────────────────────────────────────── + + +class MemorySearchRequest(BaseModel): + """Request payload for searching memory via LLMOps. + + Includes definitionSystemPrompt so LLMOps can generate the + systemPromptInjection for the agent loop. + """ + + model_config = ConfigDict(populate_by_name=True) + + fields: List[SearchField] = Field(..., alias="fields", min_length=1, max_length=20) + settings: SearchSettings = Field(..., alias="settings") + definition_system_prompt: Optional[str] = Field( + None, alias="definitionSystemPrompt" + ) + + +class MemoryMatch(BaseModel): + """A single matched memory from a search operation (LLMOps).""" + + model_config = ConfigDict(populate_by_name=True) + + memory_item_id: str = Field(..., alias="memoryItemId") + score: float = Field(..., alias="score") + semantic_score: float = Field(..., alias="semanticScore") + weighted_score: float = Field(..., alias="weightedScore") + fields: List[MemoryMatchField] = Field(..., alias="fields") + span: Optional[Any] = Field(None, alias="span") + feedback: Optional[Any] = Field(None, alias="feedback") + + +class MemorySearchResponse(BaseModel): + """Response from LLMOps search, including system prompt injection.""" + + model_config = ConfigDict(populate_by_name=True) + + results: List[MemoryMatch] = Field(default_factory=list, alias="results") + metadata: Dict[str, str] = Field(default_factory=dict, alias="metadata") + system_prompt_injection: str = Field("", alias="systemPromptInjection") + + +# ── LLMOps escalation memory models ────────────────────────────────── + + +class EscalationMemoryIngestRequest(BaseModel): + """Request payload for ingesting an escalation outcome into memory. + + Used by the escalation tool to persist resolved outcomes so + future runs can recall them without re-escalating. + """ + + model_config = ConfigDict(populate_by_name=True) + + span_id: str = Field(..., alias="spanId") + trace_id: str = Field(..., alias="traceId") + answer: str = Field(..., alias="answer") + attributes: str = Field(..., alias="attributes") + user_id: Optional[str] = Field(None, alias="userId") + + +class CachedRecall(BaseModel): + """A cached escalation answer retrieved from memory.""" + + model_config = ConfigDict(populate_by_name=True) + + output: Optional[Any] = Field(None, alias="output") + outcome: Optional[str] = Field(None, alias="outcome") + + +class EscalationMemoryMatch(BaseModel): + """A single match from an escalation memory search.""" + + model_config = ConfigDict(populate_by_name=True) + + answer: Optional[CachedRecall] = Field(None, alias="answer") + + +class EscalationMemorySearchResponse(BaseModel): + """Response from LLMOps escalation memory search.""" + + model_config = ConfigDict(populate_by_name=True) + + results: Optional[List[EscalationMemoryMatch]] = Field(None, alias="results") diff --git a/packages/uipath-platform/src/uipath/platform/orchestrator/_assets_service.py b/packages/uipath-platform/src/uipath/platform/orchestrator/_assets_service.py index 7561dd4ea..2e673fb7c 100644 --- a/packages/uipath-platform/src/uipath/platform/orchestrator/_assets_service.py +++ b/packages/uipath-platform/src/uipath/platform/orchestrator/_assets_service.py @@ -283,6 +283,56 @@ async def retrieve_async( else: return Asset.model_validate(response.json()["value"][0]) + def _resolve_robot_key( + self, + name: str, + *, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> Optional[str]: + """Return the robot key, or ``None`` if the asset opts into direct API access. + + Raises ``ValueError`` when no robot key is available and ``AllowDirectApiAccess`` + is not enabled on the asset. + """ + try: + robot_key = self._execution_context.robot_key + except ValueError: + robot_key = None + + if robot_key is None: + asset = self.retrieve( + name=name, folder_key=folder_key, folder_path=folder_path + ) + if not asset.allow_direct_api_access: + raise ValueError( + f"No robot key available and 'AllowDirectApiAccess' is disabled for asset '{name}'." + ) + return robot_key + + async def _resolve_robot_key_async( + self, + name: str, + *, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> Optional[str]: + """Async variant of :meth:`_resolve_robot_key`.""" + try: + robot_key = self._execution_context.robot_key + except ValueError: + robot_key = None + + if robot_key is None: + asset = await self.retrieve_async( + name=name, folder_key=folder_key, folder_path=folder_path + ) + if not asset.allow_direct_api_access: + raise ValueError( + f"No robot key available and 'AllowDirectApiAccess' is disabled for asset '{name}'." + ) + return robot_key + @resource_override(resource_type="asset") @traced( name="assets_credential", run_type="uipath", hide_input=True, hide_output=True @@ -294,9 +344,11 @@ def retrieve_credential( folder_key: Optional[str] = None, folder_path: Optional[str] = None, ) -> Optional[str]: - """Gets a specified Orchestrator credential. + """Get the decrypted password of a Credential asset. - The robot id is retrieved from the execution context (`UIPATH_ROBOT_KEY` environment variable) + The robot id is retrieved from the execution context (`UIPATH_ROBOT_KEY` environment variable). + If no robot key is available, the asset's `AllowDirectApiAccess` flag is checked: when + enabled, the credential is fetched without a robot key; otherwise a `ValueError` is raised. Related Activity: [Get Credential](https://docs.uipath.com/activities/other/latest/workflow/get-robot-credential) @@ -309,22 +361,18 @@ def retrieve_credential( Optional[str]: The decrypted credential password. Raises: - ValueError: If the method is called for a user asset. + ValueError: If no robot key is available and the asset does not have `AllowDirectApiAccess` enabled. """ - try: - is_user = self._execution_context.robot_key is not None - except ValueError: - is_user = False - - if not is_user: - raise ValueError("This method can only be used for robot assets.") + robot_key = self._resolve_robot_key( + name, folder_key=folder_key, folder_path=folder_path + ) - spec = self._retrieve_spec( + spec = self._retrieve_credential_spec( name, + robot_key=robot_key, folder_key=folder_key, folder_path=folder_path, ) - response = self.request( spec.method, url=spec.endpoint, @@ -333,10 +381,7 @@ def retrieve_credential( content=spec.content, headers=spec.headers, ) - - user_asset = UserAsset.model_validate(response.json()) - - return user_asset.credential_password + return UserAsset.model_validate(response.json()).credential_password @resource_override(resource_type="asset") @traced( @@ -349,9 +394,11 @@ async def retrieve_credential_async( folder_key: Optional[str] = None, folder_path: Optional[str] = None, ) -> Optional[str]: - """Asynchronously gets a specified Orchestrator credential. + """Asynchronously get the decrypted password of a Credential asset. - The robot id is retrieved from the execution context (`UIPATH_ROBOT_KEY` environment variable) + The robot id is retrieved from the execution context (`UIPATH_ROBOT_KEY` environment variable). + If no robot key is available, the asset's `AllowDirectApiAccess` flag is checked: when + enabled, the credential is fetched without a robot key; otherwise a `ValueError` is raised. Related Activity: [Get Credential](https://docs.uipath.com/activities/other/latest/workflow/get-robot-credential) @@ -364,22 +411,18 @@ async def retrieve_credential_async( Optional[str]: The decrypted credential password. Raises: - ValueError: If the method is called for a user asset. + ValueError: If no robot key is available and the asset does not have `AllowDirectApiAccess` enabled. """ - try: - is_user = self._execution_context.robot_key is not None - except ValueError: - is_user = False - - if not is_user: - raise ValueError("This method can only be used for robot assets.") + robot_key = await self._resolve_robot_key_async( + name, folder_key=folder_key, folder_path=folder_path + ) - spec = self._retrieve_spec( + spec = self._retrieve_credential_spec( name, + robot_key=robot_key, folder_key=folder_key, folder_path=folder_path, ) - response = await self.request_async( spec.method, url=spec.endpoint, @@ -388,10 +431,99 @@ async def retrieve_credential_async( content=spec.content, headers=spec.headers, ) + return UserAsset.model_validate(response.json()).credential_password + + @resource_override(resource_type="asset") + @traced(name="assets_secret", run_type="uipath", hide_input=True, hide_output=True) + def retrieve_secret( + self, + name: str, + *, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> Optional[str]: + """Get the decrypted value of a Secret asset. + + The robot id is retrieved from the execution context (`UIPATH_ROBOT_KEY` environment variable). + If no robot key is available, the asset's `AllowDirectApiAccess` flag is checked: when + enabled, the secret is fetched without a robot key; otherwise a `ValueError` is raised. - user_asset = UserAsset.model_validate(response.json()) + Args: + name (str): The name of the secret asset. + folder_key (Optional[str]): The key of the folder to execute the process in. Override the default one set in the SDK config. + folder_path (Optional[str]): The path of the folder to execute the process in. Override the default one set in the SDK config. - return user_asset.credential_password + Returns: + Optional[str]: The decrypted secret value. + + Raises: + ValueError: If no robot key is available and the asset does not have `AllowDirectApiAccess` enabled. + """ + robot_key = self._resolve_robot_key( + name, folder_key=folder_key, folder_path=folder_path + ) + + spec = self._retrieve_credential_spec( + name, + robot_key=robot_key, + folder_key=folder_key, + folder_path=folder_path, + ) + response = self.request( + spec.method, + url=spec.endpoint, + params=spec.params, + json=spec.json, + content=spec.content, + headers=spec.headers, + ) + return UserAsset.model_validate(response.json()).secret_value + + @resource_override(resource_type="asset") + @traced(name="assets_secret", run_type="uipath", hide_input=True, hide_output=True) + async def retrieve_secret_async( + self, + name: str, + *, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> Optional[str]: + """Asynchronously get the decrypted value of a Secret asset. + + The robot id is retrieved from the execution context (`UIPATH_ROBOT_KEY` environment variable). + If no robot key is available, the asset's `AllowDirectApiAccess` flag is checked: when + enabled, the secret is fetched without a robot key; otherwise a `ValueError` is raised. + + Args: + name (str): The name of the secret asset. + folder_key (Optional[str]): The key of the folder to execute the process in. Override the default one set in the SDK config. + folder_path (Optional[str]): The path of the folder to execute the process in. Override the default one set in the SDK config. + + Returns: + Optional[str]: The decrypted secret value. + + Raises: + ValueError: If no robot key is available and the asset does not have `AllowDirectApiAccess` enabled. + """ + robot_key = await self._resolve_robot_key_async( + name, folder_key=folder_key, folder_path=folder_path + ) + + spec = self._retrieve_credential_spec( + name, + robot_key=robot_key, + folder_key=folder_key, + folder_path=folder_path, + ) + response = await self.request_async( + spec.method, + url=spec.endpoint, + params=spec.params, + json=spec.json, + content=spec.content, + headers=spec.headers, + ) + return UserAsset.model_validate(response.json()).secret_value @traced(name="assets_update", run_type="uipath", hide_input=True, hide_output=True) def update( @@ -513,6 +645,32 @@ def _retrieve_spec( }, ) + def _retrieve_credential_spec( + self, + name: str, + *, + robot_key: Optional[str], + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> RequestSpec: + body: Dict[str, Any] = { + "assetName": name, + "supportsCredentialsProxyDisconnected": True, + } + if robot_key is not None: + body["robotKey"] = robot_key + + return RequestSpec( + method="POST", + endpoint=Endpoint( + "/orchestrator_/odata/Assets/UiPath.Server.Configuration.OData.GetRobotAssetByNameForRobotKey" + ), + json=body, + headers={ + **header_folder(folder_key, folder_path), + }, + ) + def _update_spec( self, robot_asset: UserAsset, diff --git a/packages/uipath-platform/src/uipath/platform/orchestrator/_attachments_service.py b/packages/uipath-platform/src/uipath/platform/orchestrator/_attachments_service.py index 73775c994..5d4c192b5 100644 --- a/packages/uipath-platform/src/uipath/platform/orchestrator/_attachments_service.py +++ b/packages/uipath-platform/src/uipath/platform/orchestrator/_attachments_service.py @@ -12,6 +12,8 @@ from httpx._types import RequestContent from uipath.core.tracing import traced +from uipath.platform.constants import TEMP_ATTACHMENTS_FOLDER + from ..attachments import Attachment, AttachmentMode, BlobFileAccessInfo from ..common._base_service import BaseService from ..common._config import UiPathApiConfig @@ -19,7 +21,6 @@ from ..common._folder_context import FolderContext, header_folder from ..common._http_config import get_httpx_client_kwargs from ..common._models import Endpoint, RequestSpec -from ..common.constants import TEMP_ATTACHMENTS_FOLDER def _upload_attachment_input_processor(inputs: dict[str, Any]) -> dict[str, Any]: diff --git a/packages/uipath-platform/src/uipath/platform/orchestrator/_buckets_service.py b/packages/uipath-platform/src/uipath/platform/orchestrator/_buckets_service.py index 0d536cd46..893435adb 100644 --- a/packages/uipath-platform/src/uipath/platform/orchestrator/_buckets_service.py +++ b/packages/uipath-platform/src/uipath/platform/orchestrator/_buckets_service.py @@ -5,6 +5,7 @@ from typing import Any, Dict, Optional, Union import httpx +from anyio import to_thread from uipath.core.tracing import traced from ..common._base_service import BaseService @@ -37,6 +38,16 @@ def __init__( self.custom_client = httpx.Client(**get_httpx_client_kwargs()) self.custom_client_async = httpx.AsyncClient(**get_httpx_client_kwargs()) + async def aclose(self) -> None: + """Close the additional HTTP clients used for bucket transfers.""" + try: + await self.custom_client_async.aclose() + finally: + try: + await to_thread.run_sync(self.custom_client.close) + finally: + await super().aclose() + @traced(name="buckets_list", run_type="uipath") def list( self, @@ -365,7 +376,7 @@ def delete( self.request( "DELETE", url=f"/orchestrator_/odata/Buckets({bucket.id})", - headers={**self.folder_headers}, + headers={**header_folder(folder_key, folder_path)}, ) @resource_override(resource_type="bucket") @@ -386,7 +397,7 @@ async def delete_async( await self.request_async( "DELETE", url=f"/orchestrator_/odata/Buckets({bucket.id})", - headers={**self.folder_headers}, + headers={**header_folder(folder_key, folder_path)}, ) @resource_override(resource_type="bucket") diff --git a/packages/uipath-platform/src/uipath/platform/orchestrator/_jobs_service.py b/packages/uipath-platform/src/uipath/platform/orchestrator/_jobs_service.py index f9433d221..fa0103b2c 100644 --- a/packages/uipath-platform/src/uipath/platform/orchestrator/_jobs_service.py +++ b/packages/uipath-platform/src/uipath/platform/orchestrator/_jobs_service.py @@ -7,13 +7,14 @@ from uipath.core.tracing import traced +from uipath.platform.constants import TEMP_ATTACHMENTS_FOLDER + from ..common._base_service import BaseService from ..common._bindings import resource_override from ..common._config import UiPathApiConfig from ..common._execution_context import UiPathExecutionContext from ..common._folder_context import FolderContext, header_folder from ..common._models import Endpoint, RequestSpec -from ..common.constants import TEMP_ATTACHMENTS_FOLDER from ..common.paging import PagedResult from ..common.validation import validate_pagination_params from ..errors import EnrichedException @@ -42,6 +43,13 @@ def __init__( self._temp_dir = os.path.join(tempfile.gettempdir(), TEMP_ATTACHMENTS_FOLDER) os.makedirs(self._temp_dir, exist_ok=True) + async def aclose(self) -> None: + """Close this service and the attachment service it owns.""" + try: + await self._attachments_service.aclose() + finally: + await super().aclose() + @overload def resume(self, *, inbox_id: str, payload: Any) -> None: ... @@ -674,6 +682,52 @@ def _retrieve_api_payload_spec( }, ) + def retrieve_inbox_payload(self, inbox_id: str) -> Any: + """Fetch payload data for Integration Services (Inbox) triggers. + + Unlike `retrieve_api_payload`, this returns the response body as-is. + Orchestrator's `GET /JobTriggers/GetPayload/{inboxId}` returns the + stored payload directly without an envelope. + + Args: + inbox_id: The Id of the inbox to fetch the payload for. + + Returns: + The stored payload. + """ + spec = self._retrieve_api_payload_spec(inbox_id=inbox_id) + + response = self.request( + spec.method, + url=spec.endpoint, + headers=spec.headers, + ) + + return response.json() + + async def retrieve_inbox_payload_async(self, inbox_id: str) -> Any: + """Asynchronously fetch payload data for Integration Services (Inbox) triggers. + + Unlike `retrieve_api_payload_async`, this returns the response body + as-is. Orchestrator's `GET /JobTriggers/GetPayload/{inboxId}` returns + the stored payload directly without an envelope. + + Args: + inbox_id: The Id of the inbox to fetch the payload for. + + Returns: + The stored payload. + """ + spec = self._retrieve_api_payload_spec(inbox_id=inbox_id) + + response = await self.request_async( + spec.method, + url=spec.endpoint, + headers=spec.headers, + ) + + return response.json() + def _extract_first_inbox_id(self, response: Any) -> str: if len(response["value"]) > 0: return response["value"][0]["ItemKey"] diff --git a/packages/uipath-platform/src/uipath/platform/orchestrator/_mcp_service.py b/packages/uipath-platform/src/uipath/platform/orchestrator/_mcp_service.py index 195eb8240..e2a9f13c3 100644 --- a/packages/uipath-platform/src/uipath/platform/orchestrator/_mcp_service.py +++ b/packages/uipath-platform/src/uipath/platform/orchestrator/_mcp_service.py @@ -1,4 +1,5 @@ from typing import List +from urllib.parse import quote from uipath.core.tracing import traced @@ -8,6 +9,7 @@ from ..common._execution_context import UiPathExecutionContext from ..common._folder_context import FolderContext, header_folder from ..common._models import Endpoint, RequestSpec +from ..common._resource_identifier import resolve_retrieve_identifier from ._folder_service import FolderService from .mcp import McpServer @@ -66,7 +68,7 @@ def list( return [McpServer.model_validate(server) for server in response.json()] - @traced(name="mcp_list", run_type="uipath") + @traced(name="mcp_list_async", run_type="uipath") async def list_async( self, *, @@ -109,18 +111,21 @@ async def main(): return [McpServer.model_validate(server) for server in response.json()] + @resource_override(resource_type="mcpServer", resource_identifier="name") @resource_override(resource_type="mcpServer", resource_identifier="slug") @traced(name="mcp_retrieve", run_type="uipath") def retrieve( self, - slug: str, + slug: str | None = None, *, + name: str | None = None, folder_path: str | None = None, ) -> McpServer: - """Retrieve a specific MCP server by its slug. + """Retrieve a specific MCP server by its display name or legacy slug. Args: - slug (str): The unique slug identifier for the server. + slug (Optional[str]): The legacy slug identifier of the server. + name (Optional[str]): The display name of the server. folder_path (Optional[str]): The path of the folder where the server is located. Returns: @@ -132,12 +137,13 @@ def retrieve( client = UiPath() - server = client.mcp.retrieve(slug="my-server-slug", folder_path="MyFolder") + server = client.mcp.retrieve(name="My Server", folder_path="MyFolder") print(f"Server: {server.name}, URL: {server.mcp_url}") ``` """ + identifier = resolve_retrieve_identifier(name=name, slug=slug) spec = self._retrieve_spec( - slug=slug, + name=identifier, folder_path=folder_path, ) @@ -150,18 +156,21 @@ def retrieve( return McpServer.model_validate(response.json()) + @resource_override(resource_type="mcpServer", resource_identifier="name") @resource_override(resource_type="mcpServer", resource_identifier="slug") - @traced(name="mcp_retrieve", run_type="uipath") + @traced(name="mcp_retrieve_async", run_type="uipath") async def retrieve_async( self, - slug: str, + slug: str | None = None, *, + name: str | None = None, folder_path: str | None = None, ) -> McpServer: - """Asynchronously retrieve a specific MCP server by its slug. + """Asynchronously retrieve an MCP server by its display name or legacy slug. Args: - slug (str): The unique slug identifier for the server. + slug (Optional[str]): The legacy slug identifier of the server. + name (Optional[str]): The display name of the server. folder_path (Optional[str]): The path of the folder where the server is located. Returns: @@ -176,14 +185,15 @@ async def retrieve_async( sdk = UiPath() async def main(): - server = await sdk.mcp.retrieve_async(slug="my-server-slug", folder_path="MyFolder") + server = await sdk.mcp.retrieve_async(name="My Server", folder_path="MyFolder") print(f"Server: {server.name}, URL: {server.mcp_url}") asyncio.run(main()) ``` """ + identifier = resolve_retrieve_identifier(name=name, slug=slug) spec = self._retrieve_spec( - slug=slug, + name=identifier, folder_path=folder_path, ) @@ -223,14 +233,14 @@ def _list_spec( def _retrieve_spec( self, - slug: str, + name: str, *, folder_path: str | None, ) -> RequestSpec: folder_key = self._resolve_folder_key(folder_path) return RequestSpec( method="GET", - endpoint=Endpoint(f"/agenthub_/api/servers/{slug}"), + endpoint=Endpoint(f"/agenthub_/api/servers/{quote(name, safe='')}"), headers={ **header_folder(folder_key, None), }, diff --git a/packages/uipath-platform/src/uipath/platform/orchestrator/_processes_service.py b/packages/uipath-platform/src/uipath/platform/orchestrator/_processes_service.py index 10b6010e2..433793389 100644 --- a/packages/uipath-platform/src/uipath/platform/orchestrator/_processes_service.py +++ b/packages/uipath-platform/src/uipath/platform/orchestrator/_processes_service.py @@ -7,6 +7,8 @@ from opentelemetry.trace import format_span_id from uipath.core.tracing import traced +from uipath.platform.constants import ENV_JOB_KEY, HEADER_JOB_KEY + from ..attachments import Attachment from ..common._base_service import BaseService from ..common._bindings import resource_override @@ -15,7 +17,6 @@ from ..common._folder_context import FolderContext, header_folder from ..common._models import Endpoint, RequestSpec from ..common._span_utils import _SpanUtils -from ..common.constants import ENV_JOB_KEY, HEADER_JOB_KEY from ._attachments_service import AttachmentsService from .job import Job @@ -50,6 +51,7 @@ def invoke( folder_path: Optional[str] = None, attachments: Optional[list[Attachment]] = None, parent_operation_id: Optional[str] = None, + run_as_me: Optional[bool] = None, **kwargs: Any, ) -> Job: """Start execution of a process by its name. @@ -63,6 +65,7 @@ def invoke( folder_key (Optional[str]): The key of the folder to execute the process in. Override the default one set in the SDK config. folder_path (Optional[str]): The path of the folder to execute the process in. Override the default one set in the SDK config. parent_operation_id (Optional[str]): The parent operation ID for BTS tracking correlation. + run_as_me (Optional[bool]): If True, the job will run under the calling user's identity. Returns: Job: The job execution details. @@ -100,6 +103,7 @@ def invoke( folder_path=folder_path, parent_span_id=kwargs.get("parent_span_id"), parent_operation_id=parent_operation_id, + run_as_me=run_as_me, ) response = self.request( spec.method, @@ -123,6 +127,7 @@ async def invoke_async( folder_path: Optional[str] = None, attachments: Optional[list[Attachment]] = None, parent_operation_id: Optional[str] = None, + run_as_me: Optional[bool] = None, **kwargs: Any, ) -> Job: """Asynchronously start execution of a process by its name. @@ -136,6 +141,7 @@ async def invoke_async( folder_key (Optional[str]): The key of the folder to execute the process in. Override the default one set in the SDK config. folder_path (Optional[str]): The path of the folder to execute the process in. Override the default one set in the SDK config. parent_operation_id (Optional[str]): The parent operation ID for BTS tracking correlation. + run_as_me (Optional[bool]): If True, the job will run under the calling user's identity. Returns: Job: The job execution details. @@ -168,6 +174,7 @@ async def main(): folder_path=folder_path, parent_span_id=kwargs.get("parent_span_id"), parent_operation_id=parent_operation_id, + run_as_me=run_as_me, ) response = await self.request_async( @@ -313,13 +320,21 @@ def _invoke_spec( folder_path: Optional[str] = None, parent_span_id: Optional[str] = None, parent_operation_id: Optional[str] = None, + run_as_me: Optional[bool] = None, ) -> RequestSpec: - payload: Dict[str, Any] = {"ReleaseName": name, **(input_data or {})} + payload: Dict[str, Any] = { + "ReleaseName": name, + **(input_data or {}), + "Source": "AgentService", + } self._add_tracing(payload, UiPathConfig.trace_id, parent_span_id) if parent_operation_id: payload["ParentOperationId"] = parent_operation_id + if run_as_me is not None: + payload["RunAsMe"] = run_as_me + request_spec = RequestSpec( method="POST", endpoint=Endpoint( diff --git a/packages/uipath-platform/src/uipath/platform/orchestrator/_queues_service.py b/packages/uipath-platform/src/uipath/platform/orchestrator/_queues_service.py index eede00ecf..1a2985072 100644 --- a/packages/uipath-platform/src/uipath/platform/orchestrator/_queues_service.py +++ b/packages/uipath-platform/src/uipath/platform/orchestrator/_queues_service.py @@ -454,7 +454,9 @@ def _create_item_spec( elif isinstance(item, QueueItem): queue_item = item - item_data = queue_item.model_dump(exclude_unset=True, by_alias=True) + item_data = queue_item.model_dump( + mode="json", exclude_unset=True, by_alias=True + ) resolved_name = queue_name or item_data.get("Name") if resolved_name is None: raise ValueError( @@ -493,9 +495,11 @@ def _create_items_spec( "queueName": queue_name, "commitType": commit_type.value, "queueItems": [ - item.model_dump(exclude_unset=True, by_alias=True) + item.model_dump(mode="json", exclude_unset=True, by_alias=True) if isinstance(item, QueueItem) - else QueueItem(**item).model_dump(exclude_unset=True, by_alias=True) + else QueueItem(**item).model_dump( + mode="json", exclude_unset=True, by_alias=True + ) for item in items ], }, @@ -519,7 +523,7 @@ def _create_transaction_item_spec( transaction_item = item transaction_data = transaction_item.model_dump( - exclude_unset=True, by_alias=True + mode="json", exclude_unset=True, by_alias=True ) resolved_name = queue_name or transaction_data.get("Name") if resolved_name is None: @@ -580,7 +584,7 @@ def _complete_transaction_item_spec( ), json={ "transactionResult": transaction_result.model_dump( - exclude_unset=True, by_alias=True + mode="json", exclude_unset=True, by_alias=True ) }, headers={ diff --git a/packages/uipath-platform/src/uipath/platform/orchestrator/assets.py b/packages/uipath-platform/src/uipath/platform/orchestrator/assets.py index 6ee89e806..122821029 100644 --- a/packages/uipath-platform/src/uipath/platform/orchestrator/assets.py +++ b/packages/uipath-platform/src/uipath/platform/orchestrator/assets.py @@ -38,6 +38,7 @@ class UserAsset(BaseModel): int_value: Optional[int] = Field(default=None, alias="IntValue") credential_username: Optional[str] = Field(default=None, alias="CredentialUsername") credential_password: Optional[str] = Field(default=None, alias="CredentialPassword") + secret_value: Optional[str] = Field(default=None, alias="SecretValue") external_name: Optional[str] = Field(default=None, alias="ExternalName") credential_store_id: Optional[int] = Field(default=None, alias="CredentialStoreId") key_value_list: Optional[List[Dict[str, str]]] = Field( @@ -46,6 +47,9 @@ class UserAsset(BaseModel): connection_data: Optional[CredentialsConnectionData] = Field( default=None, alias="ConnectionData" ) + allow_direct_api_access: Optional[bool] = Field( + default=None, alias="AllowDirectApiAccess" + ) id: Optional[int] = Field(default=None, alias="Id") @@ -69,5 +73,9 @@ class Asset(BaseModel): int_value: Optional[int] = Field(default=None, alias="IntValue") credential_username: Optional[str] = Field(default=None, alias="CredentialUsername") credential_password: Optional[str] = Field(default=None, alias="CredentialPassword") + secret_value: Optional[str] = Field(default=None, alias="SecretValue") external_name: Optional[str] = Field(default=None, alias="ExternalName") credential_store_id: Optional[int] = Field(default=None, alias="CredentialStoreId") + allow_direct_api_access: Optional[bool] = Field( + default=None, alias="AllowDirectApiAccess" + ) diff --git a/packages/uipath-platform/src/uipath/platform/orchestrator/job.py b/packages/uipath-platform/src/uipath/platform/orchestrator/job.py index 7ade631e5..6464405b4 100644 --- a/packages/uipath-platform/src/uipath/platform/orchestrator/job.py +++ b/packages/uipath-platform/src/uipath/platform/orchestrator/job.py @@ -79,5 +79,5 @@ class Job(BaseModel): has_errors: Optional[bool] = Field(default=None, alias="HasErrors") has_warnings: Optional[bool] = Field(default=None, alias="HasWarnings") job_error: Optional[JobErrorInfo] = Field(default=None, alias="JobError") - folder_key: str = Field(alias="FolderKey") + folder_key: Optional[str] = Field(default=None, alias="FolderKey") id: int = Field(alias="Id") diff --git a/packages/uipath-platform/src/uipath/platform/orchestrator/mcp.py b/packages/uipath-platform/src/uipath/platform/orchestrator/mcp.py index 9a811d876..ca2050da3 100644 --- a/packages/uipath-platform/src/uipath/platform/orchestrator/mcp.py +++ b/packages/uipath-platform/src/uipath/platform/orchestrator/mcp.py @@ -17,6 +17,8 @@ class McpServerType(IntEnum): SelfHosted = 3 # tunnel to (externally) self-hosted server Remote = 4 # HTTP connection to remote MCP server ProcessAssistant = 5 # Dynamic user process assistant + Platform = 6 # Platform MCP server (e.g: Orchestrator, TestManager) + Swagger = 7 # User-provided Swagger/OpenAPI spec exposed as MCP server class McpServerStatus(IntEnum): diff --git a/packages/uipath-platform/src/uipath/platform/pii_detection/__init__.py b/packages/uipath-platform/src/uipath/platform/pii_detection/__init__.py new file mode 100644 index 000000000..7ab3b9e26 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/pii_detection/__init__.py @@ -0,0 +1,36 @@ +"""PiiDetection service package. + +Provides the ``PiiDetectionService`` client, Pydantic request/response models for +the PII detection endpoint, and utilities for rehydrating masked text with +original PII values after LLM processing. +""" + +from ._pii_detection_service import PiiDetectionService +from .pii_detection import ( + PiiDetectionRequest, + PiiDetectionResponse, + PiiDocument, + PiiDocumentResult, + PiiEntity, + PiiEntityThreshold, + PiiFile, + PiiFileResult, +) +from .pii_utilities import ( + rehydrate_from_pii_entities, + rehydrate_from_pii_response, +) + +__all__ = [ + "PiiDetectionRequest", + "PiiDetectionResponse", + "PiiDetectionService", + "PiiDocument", + "PiiDocumentResult", + "PiiEntity", + "PiiEntityThreshold", + "PiiFile", + "PiiFileResult", + "rehydrate_from_pii_entities", + "rehydrate_from_pii_response", +] diff --git a/packages/uipath-platform/src/uipath/platform/pii_detection/_pii_detection_service.py b/packages/uipath-platform/src/uipath/platform/pii_detection/_pii_detection_service.py new file mode 100644 index 000000000..a39ed4196 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/pii_detection/_pii_detection_service.py @@ -0,0 +1,80 @@ +"""PiiDetection service for UiPath Platform. + +Provides methods for detecting PII in documents and files. +""" + +from uipath.core.tracing import traced + +from ..common._base_service import BaseService +from ..common._config import UiPathApiConfig +from ..common._execution_context import UiPathExecutionContext +from ..common._models import Endpoint, RequestSpec +from .pii_detection import PiiDetectionRequest, PiiDetectionResponse + +_PII_DETECTION_ENDPOINT = Endpoint("llmopstenant_/api/pii-detection") + +# PII detection over documents/files can be slow, so override the default +# httpx client timeout (30s) with a longer per-request timeout. +_PII_DETECTION_TIMEOUT = 290.0 + + +class PiiDetectionService(BaseService): + """Service for detecting PII via UiPath.""" + + def __init__( + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + ) -> None: + super().__init__(config=config, execution_context=execution_context) + + @traced(name="pii_detection_detect_pii", run_type="uipath") + def detect_pii(self, request: PiiDetectionRequest) -> PiiDetectionResponse: + """Detect PII in the provided documents and/or files. + + Args: + request: The PII detection request payload. + + Returns: + The PII detection response. + """ + spec = self._pii_detection_spec(request) + response = self.request( + spec.method, + url=spec.endpoint, + json=spec.json, + headers=spec.headers, + scoped="tenant", + timeout=_PII_DETECTION_TIMEOUT, + ) + return PiiDetectionResponse.model_validate(response.json()) + + @traced(name="pii_detection_detect_pii", run_type="uipath") + async def detect_pii_async( + self, request: PiiDetectionRequest + ) -> PiiDetectionResponse: + """Detect PII in the provided documents and/or files (async). + + Args: + request: The PII detection request payload. + + Returns: + The PII detection response. + """ + spec = self._pii_detection_spec(request) + response = await self.request_async( + spec.method, + url=spec.endpoint, + json=spec.json, + headers=spec.headers, + scoped="tenant", + timeout=_PII_DETECTION_TIMEOUT, + ) + return PiiDetectionResponse.model_validate(response.json()) + + def _pii_detection_spec(self, request: PiiDetectionRequest) -> RequestSpec: + return RequestSpec( + method="POST", + endpoint=_PII_DETECTION_ENDPOINT, + json=request.model_dump(by_alias=True, exclude_none=True), + ) diff --git a/packages/uipath-platform/src/uipath/platform/pii_detection/pii_detection.py b/packages/uipath-platform/src/uipath/platform/pii_detection/pii_detection.py new file mode 100644 index 000000000..94ac10fca --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/pii_detection/pii_detection.py @@ -0,0 +1,91 @@ +"""Public Pydantic models for the PiiDetection service.""" + +from typing import Optional + +from pydantic import BaseModel, ConfigDict, Field + + +class PiiDocument(BaseModel): + """A text document to scan for PII.""" + + id: str + role: str + document: str + + +class PiiFile(BaseModel): + """A file reference to scan for PII.""" + + model_config = ConfigDict(populate_by_name=True) + + file_name: str = Field(alias="fileName") + file_url: str = Field(alias="fileUrl") + file_type: str = Field(alias="fileType") + + +class PiiEntityThreshold(BaseModel): + """Per-entity confidence threshold override.""" + + model_config = ConfigDict(populate_by_name=True) + + category: str = Field(alias="pii-entity-category") + confidence_threshold: float = Field(alias="pii-entity-confidence-threshold") + + +class PiiDetectionRequest(BaseModel): + """Request payload for the PII detection endpoint.""" + + model_config = ConfigDict(populate_by_name=True) + + documents: Optional[list[PiiDocument]] = None + files: Optional[list[PiiFile]] = None + language_code: Optional[str] = Field(default=None, alias="languageCode") + confidence_threshold: Optional[float] = Field( + default=None, alias="confidenceThreshold" + ) + entity_thresholds: Optional[list[PiiEntityThreshold]] = Field( + default=None, alias="entityThresholds" + ) + + +class PiiEntity(BaseModel): + """A single detected PII entity.""" + + model_config = ConfigDict(populate_by_name=True) + + pii_text: str = Field(alias="piiText") + replacement_text: str = Field(alias="replacementText") + pii_type: str = Field(alias="piiType") + offset: int + confidence_score: float = Field(alias="confidenceScore") + + +class PiiDocumentResult(BaseModel): + """PII detection result for a single document.""" + + model_config = ConfigDict(populate_by_name=True) + + id: str + role: str + masked_document: str = Field(alias="maskedDocument") + initial_document: str = Field(alias="initialDocument") + pii_entities: list[PiiEntity] = Field(default_factory=list, alias="piiEntities") + + +class PiiFileResult(BaseModel): + """PII detection result for a single file (fileUrl is the redacted URL).""" + + model_config = ConfigDict(populate_by_name=True) + + file_name: str = Field(alias="fileName") + file_url: str = Field(alias="fileUrl") + pii_entities: list[PiiEntity] = Field(default_factory=list, alias="piiEntities") + + +class PiiDetectionResponse(BaseModel): + """Response payload from the PII detection endpoint.""" + + model_config = ConfigDict(populate_by_name=True) + + response: list[PiiDocumentResult] = Field(default_factory=list) + files: list[PiiFileResult] = Field(default_factory=list) diff --git a/packages/uipath-platform/src/uipath/platform/pii_detection/pii_utilities.py b/packages/uipath-platform/src/uipath/platform/pii_detection/pii_utilities.py new file mode 100644 index 000000000..b2fa482d0 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/pii_detection/pii_utilities.py @@ -0,0 +1,98 @@ +"""Utility methods for working with PII data. + +Python port of UiPath.SemanticProxy.Client.PiiUtilities (C#). +""" + +import re +from typing import Callable, Iterable + +from .pii_detection import PiiDetectionResponse, PiiEntity + + +def rehydrate_from_pii_entities( + masked_text: str, pii_entities: Iterable[PiiEntity] +) -> str: + """Rehydrate masked text by replacing PII placeholders with original values. + + Placeholders (e.g. ``[Person-1]``) are matched case-insensitively and replaced + with the corresponding original PII text. The function also replaces variants + without the surrounding brackets (e.g. ``Person-1``) in case the LLM stripped + them in its output. + + Args: + masked_text: The masked text with PII placeholders. + pii_entities: The PII entities containing the original values. + + Returns: + The rehydrated text with original PII values. + """ + if not masked_text: + return masked_text + + entities = [e for e in pii_entities if e.replacement_text] + if not entities: + return masked_text + + # Sort by replacement text length descending to avoid substring collisions + # (e.g. "[Person-10]" must be replaced before "[Person-1]"). + entities.sort(key=lambda e: len(e.replacement_text), reverse=True) + + rehydrated = masked_text + for entity in entities: + if not entity.replacement_text or not entity.pii_text: + continue + # Replace the full placeholder (with brackets) case-insensitively. + # ``_literal_replacer`` bypasses regex backreference interpretation in the + # replacement string. + rehydrated = re.sub( + re.escape(entity.replacement_text), + _literal_replacer(entity.pii_text), + rehydrated, + flags=re.IGNORECASE, + ) + # Also replace the content without brackets (in case the LLM dropped them). + if entity.replacement_text.startswith("[") and entity.replacement_text.endswith( + "]" + ): + no_brackets = entity.replacement_text[1:-1] + rehydrated = re.sub( + re.escape(no_brackets), + _literal_replacer(entity.pii_text), + rehydrated, + flags=re.IGNORECASE, + ) + + return rehydrated + + +def _literal_replacer(replacement: str) -> Callable[[re.Match[str]], str]: + """Return a replacement function that ignores regex backreference syntax.""" + + def replace(_match: re.Match[str]) -> str: + return replacement + + return replace + + +def rehydrate_from_pii_response( + masked_text: str, response: PiiDetectionResponse +) -> str: + """Rehydrate masked text using all PII entities from a detection response. + + Merges entities from both ``response.response`` (detected in documents/prompts) + and ``response.files`` (detected in files), so placeholders originating from + either source are rehydrated. + + Args: + masked_text: The masked text with PII placeholders. + response: The PII detection response containing entities to rehydrate. + + Returns: + The rehydrated text with original PII values. + """ + entities: list[PiiEntity] = [] + for doc in response.response: + entities.extend(doc.pii_entities) + for file in response.files: + entities.extend(file.pii_entities) + return rehydrate_from_pii_entities(masked_text, entities) diff --git a/packages/uipath-platform/src/uipath/platform/resource_catalog/_resource_catalog_service.py b/packages/uipath-platform/src/uipath/platform/resource_catalog/_resource_catalog_service.py index 030d85d01..e42880e71 100644 --- a/packages/uipath-platform/src/uipath/platform/resource_catalog/_resource_catalog_service.py +++ b/packages/uipath-platform/src/uipath/platform/resource_catalog/_resource_catalog_service.py @@ -1,4 +1,4 @@ -from typing import Any, AsyncIterator, Dict, Iterator, List, Optional +from typing import Any, AsyncGenerator, Dict, Iterator, List, Optional from uipath.core.tracing import traced @@ -110,7 +110,7 @@ async def search_async( resource_types: Optional[List[ResourceType]] = None, resource_sub_types: Optional[List[str]] = None, page_size: int = _DEFAULT_PAGE_SIZE, - ) -> AsyncIterator[Resource]: + ) -> AsyncGenerator[Resource, None]: """Asynchronously search for tenant scoped resources and folder scoped resources (accessible to the user). This method automatically handles pagination and yields resources one by one. @@ -258,7 +258,7 @@ async def list_async( folder_path: Optional[str] = None, folder_key: Optional[str] = None, page_size: int = _DEFAULT_PAGE_SIZE, - ) -> AsyncIterator[Resource]: + ) -> AsyncGenerator[Resource, None]: """Asynchronously get tenant scoped resources and folder scoped resources (accessible to the user). If no folder identifier is provided (path or key) only tenant resources will be retrieved. @@ -428,7 +428,7 @@ async def list_by_type_async( folder_path: Optional[str] = None, folder_key: Optional[str] = None, page_size: int = _DEFAULT_PAGE_SIZE, - ) -> AsyncIterator[Resource]: + ) -> AsyncGenerator[Resource, None]: """Asynchronously get resources of a specific type (tenant scoped or folder scoped). If no folder identifier is provided (path or key) only tenant resources will be retrieved. diff --git a/packages/uipath-platform/src/uipath/platform/resource_catalog/resource_catalog.py b/packages/uipath-platform/src/uipath/platform/resource_catalog/resource_catalog.py index bedf6525d..67fdf52f6 100644 --- a/packages/uipath-platform/src/uipath/platform/resource_catalog/resource_catalog.py +++ b/packages/uipath-platform/src/uipath/platform/resource_catalog/resource_catalog.py @@ -22,6 +22,7 @@ class ResourceType(str, Enum): CONNECTOR = "connector" MCP_SERVER = "mcpserver" QUEUE = "queue" + ENTITY = "entity" @classmethod def from_string(cls, value: str) -> "ResourceType": diff --git a/packages/uipath-platform/src/uipath/platform/resume_triggers/_protocol.py b/packages/uipath-platform/src/uipath/platform/resume_triggers/_protocol.py index 60a169da9..97dd5731a 100644 --- a/packages/uipath-platform/src/uipath/platform/resume_triggers/_protocol.py +++ b/packages/uipath-platform/src/uipath/platform/resume_triggers/_protocol.py @@ -3,6 +3,7 @@ import json import os import uuid +from functools import cache from typing import Any from uipath.core.errors import ( @@ -13,6 +14,7 @@ from uipath.core.serialization import serialize_object from uipath.core.triggers import ( UiPathApiTrigger, + UiPathIntegrationTrigger, UiPathResumeTrigger, UiPathResumeTriggerName, UiPathResumeTriggerType, @@ -43,17 +45,22 @@ WaitEphemeralIndex, WaitEphemeralIndexRaw, WaitEscalation, + WaitIntegrationEvent, WaitJob, WaitJobRaw, WaitSystemAgent, WaitTask, + WaitUntil, ) +from uipath.platform.connections import EventArguments from uipath.platform.context_grounding import DeepRagStatus, IndexStatus from uipath.platform.context_grounding.context_grounding_index import ( ContextGroundingIndex, ) from uipath.platform.errors import ( + BatchTransformFailedException, BatchTransformNotCompleteException, + ContextGroundingIndexNotFoundError, OperationNotCompleteException, ) from uipath.platform.orchestrator.job import JobState @@ -125,12 +132,18 @@ async def read_trigger(self, trigger: UiPathResumeTrigger) -> Any | None: UiPathRuntimeError: If reading fails, job failed, API connection failed, trigger type is unknown, or HITL feedback retrieval failed. """ - uipath = UiPath() + + @cache + def get_uipath() -> UiPath: + return UiPath() match trigger.trigger_type: + case UiPathResumeTriggerType.TIMER: + return {"resumeTime": serialize_object(trigger.resume_time)} + case UiPathResumeTriggerType.TASK: if trigger.item_key: - task: Task = await uipath.tasks.retrieve_async( + task: Task = await get_uipath().tasks.retrieve_async( trigger.item_key, app_folder_key=trigger.folder_key, app_folder_path=trigger.folder_path, @@ -178,7 +191,7 @@ async def read_trigger(self, trigger: UiPathResumeTrigger) -> Any | None: case UiPathResumeTriggerType.JOB: if trigger.item_key: - job = await uipath.jobs.retrieve_async( + job = await get_uipath().jobs.retrieve_async( trigger.item_key, folder_key=trigger.folder_key, folder_path=trigger.folder_path, @@ -219,7 +232,7 @@ async def read_trigger(self, trigger: UiPathResumeTrigger) -> Any | None: f"Process did not finish successfully. Error: {job_error}", ) - output_data = await uipath.jobs.extract_output_async(job) + output_data = await get_uipath().jobs.extract_output_async(job) trigger_response = _try_convert_to_json_format(output_data) # if response is an empty dictionary, use job state as placeholder value @@ -235,9 +248,13 @@ async def read_trigger(self, trigger: UiPathResumeTrigger) -> Any | None: return trigger_response case UiPathResumeTriggerType.DEEP_RAG: if trigger.item_key: - deep_rag = await uipath.context_grounding.retrieve_deep_rag_async( - trigger.item_key, - index_name=self._extract_field("index_name", trigger.payload), + deep_rag = ( + await get_uipath().context_grounding.retrieve_deep_rag_async( + trigger.item_key, + index_name=self._extract_field( + "index_name", trigger.payload + ), + ) ) deep_rag_status = deep_rag.last_deep_rag_status @@ -275,7 +292,7 @@ async def read_trigger(self, trigger: UiPathResumeTrigger) -> Any | None: case UiPathResumeTriggerType.INDEX_INGESTION: if trigger.item_key: - index = await uipath.context_grounding.retrieve_by_id_async( + index = await get_uipath().context_grounding.retrieve_by_id_async( trigger.item_key ) @@ -315,7 +332,7 @@ async def read_trigger(self, trigger: UiPathResumeTrigger) -> Any | None: ) assert destination_path is not None try: - await uipath.context_grounding.download_batch_transform_result_async( + await get_uipath().context_grounding.download_batch_transform_result_async( trigger.item_key, destination_path, validate_status=True, @@ -323,6 +340,11 @@ async def read_trigger(self, trigger: UiPathResumeTrigger) -> Any | None: "index_name", trigger.payload ), ) + except BatchTransformFailedException as e: + raise UiPathFaultedTriggerError( + ErrorCategory.SYSTEM, + f"{e.message}", + ) from e except BatchTransformNotCompleteException as e: raise UiPathPendingTriggerError( ErrorCategory.SYSTEM, @@ -340,10 +362,8 @@ async def read_trigger(self, trigger: UiPathResumeTrigger) -> Any | None: assert tag is not None try: - extraction_response = ( - await uipath.documents.retrieve_ixp_extraction_result_async( - project_id, tag, trigger.item_key - ) + extraction_response = await get_uipath().documents.retrieve_ixp_extraction_result_async( + project_id, tag, trigger.item_key ) except OperationNotCompleteException as e: raise UiPathPendingTriggerError( @@ -361,7 +381,7 @@ async def read_trigger(self, trigger: UiPathResumeTrigger) -> Any | None: assert project_id is not None assert tag is not None try: - escalation_response = await uipath.documents.retrieve_ixp_extraction_validation_result_async( + escalation_response = await get_uipath().documents.retrieve_ixp_extraction_validation_result_async( project_id, tag, trigger.item_key ) except OperationNotCompleteException as e: @@ -385,7 +405,7 @@ async def read_trigger(self, trigger: UiPathResumeTrigger) -> Any | None: case UiPathResumeTriggerType.API: if trigger.api_resume and trigger.api_resume.inbox_id: try: - return await uipath.jobs.retrieve_api_payload_async( + return await get_uipath().jobs.retrieve_api_payload_async( trigger.api_resume.inbox_id ) except Exception as e: @@ -395,6 +415,27 @@ async def read_trigger(self, trigger: UiPathResumeTrigger) -> Any | None: f"Error fetching API trigger payload for inbox {trigger.api_resume.inbox_id}: {str(e)}", ) from e + case UiPathResumeTriggerType.INBOX: + if trigger.integration_resume and trigger.integration_resume.inbox_id: + try: + inbox_payload = ( + await get_uipath().jobs.retrieve_inbox_payload_async( + trigger.integration_resume.inbox_id + ) + ) + event_args = EventArguments.model_validate(inbox_payload) + return ( + await get_uipath().connections.retrieve_event_payload_async( + event_args + ) + ) + except Exception as e: + raise UiPathFaultedTriggerError( + ErrorCategory.SYSTEM, + f"Failed to get trigger payload" + f"Error fetching Inbox trigger payload for inbox {trigger.integration_resume.inbox_id}: {str(e)}", + ) from e + case _: raise UiPathFaultedTriggerError( ErrorCategory.SYSTEM, @@ -413,6 +454,23 @@ class UiPathResumeTriggerCreator: Implements UiPathResumeTriggerCreatorProtocol. """ + async def create_triggers(self, suspend_value: Any) -> list[UiPathResumeTrigger]: + """Create resume triggers from a suspend value. + + Most values create a single trigger. A list or tuple creates sibling + triggers for the same interrupt; whichever one fires first resumes it. + """ + if isinstance(suspend_value, (list, tuple)): + if not suspend_value: + raise ValueError("At least one interrupt model is required.") + return [ + await self.create_trigger(child_suspend_value) + for child_suspend_value in suspend_value + ] + + resume_trigger = await self.create_trigger(suspend_value) + return [resume_trigger] + async def create_trigger(self, suspend_value: Any) -> UiPathResumeTrigger: """Create a resume trigger from a suspend value. @@ -455,6 +513,12 @@ async def create_trigger(self, suspend_value: Any) -> UiPathResumeTrigger: case UiPathResumeTriggerType.API: self._handle_api_trigger(suspend_value, resume_trigger) + case UiPathResumeTriggerType.INBOX: + await self._handle_inbox_trigger(suspend_value, resume_trigger) + + case UiPathResumeTriggerType.TIMER: + self._handle_time_trigger(suspend_value, resume_trigger) + case UiPathResumeTriggerType.DEEP_RAG: await self._handle_deep_rag_job_trigger( suspend_value, resume_trigger @@ -481,6 +545,8 @@ async def create_trigger(self, suspend_value: Any) -> UiPathResumeTrigger: f"Unexpected model received" f"{type(suspend_value)} is not a valid Human-In-The-Loop model", ) + except UiPathFaultedTriggerError: + raise except Exception as e: raise UiPathFaultedTriggerError( ErrorCategory.SYSTEM, @@ -539,6 +605,10 @@ def _determine_trigger_type(self, value: Any) -> UiPathResumeTriggerType: value, (DocumentExtractionValidation, WaitDocumentExtractionValidation) ): return UiPathResumeTriggerType.IXP_VS_ESCALATION + if isinstance(value, WaitIntegrationEvent): + return UiPathResumeTriggerType.INBOX + if isinstance(value, WaitUntil): + return UiPathResumeTriggerType.TIMER # default to API trigger return UiPathResumeTriggerType.API @@ -573,6 +643,10 @@ def _determine_trigger_name(self, value: Any) -> UiPathResumeTriggerName: return UiPathResumeTriggerName.BATCH_RAG if isinstance(value, (DocumentExtraction, WaitDocumentExtraction)): return UiPathResumeTriggerName.EXTRACTION + if isinstance(value, WaitIntegrationEvent): + return UiPathResumeTriggerName.INBOX + if isinstance(value, WaitUntil): + return UiPathResumeTriggerName.TIMER # default to API trigger return UiPathResumeTriggerName.API @@ -626,28 +700,35 @@ async def _handle_deep_rag_job_trigger( resume_trigger.item_key = value.deep_rag.id elif isinstance(value, CreateDeepRag): uipath = UiPath() - if value.is_ephemeral_index: - deep_rag = ( - await uipath.context_grounding.start_deep_rag_ephemeral_async( + try: + if value.is_ephemeral_index: + deep_rag = ( + await uipath.context_grounding.start_deep_rag_ephemeral_async( + name=value.name, + index_id=value.index_id, + prompt=value.prompt, + glob_pattern=value.glob_pattern, + citation_mode=value.citation_mode, + ) + ) + else: + deep_rag = await uipath.context_grounding.start_deep_rag_async( name=value.name, + index_name=value.index_name, index_id=value.index_id, prompt=value.prompt, glob_pattern=value.glob_pattern, citation_mode=value.citation_mode, + folder_path=value.index_folder_path, + folder_key=value.index_folder_key, ) - ) - - else: - deep_rag = await uipath.context_grounding.start_deep_rag_async( - name=value.name, - index_name=value.index_name, - index_id=value.index_id, - prompt=value.prompt, - glob_pattern=value.glob_pattern, - citation_mode=value.citation_mode, - folder_path=value.index_folder_path, - folder_key=value.index_folder_key, - ) + except ContextGroundingIndexNotFoundError as e: + raise UiPathFaultedTriggerError( + ErrorCategory.DEPLOYMENT, + "Context grounding index not found. Check that the index is " + "deployed and available in the configured folder.", + str(e), + ) from e if not deep_rag: raise Exception("Failed to start deep rag") @@ -707,27 +788,35 @@ async def _handle_batch_rag_job_trigger( resume_trigger.item_key = value.batch_transform.id elif isinstance(value, CreateBatchTransform): uipath = UiPath() - if value.is_ephemeral_index: - batch_transform = await uipath.context_grounding.start_batch_transform_ephemeral_async( - name=value.name, - index_id=value.index_id, - prompt=value.prompt, - output_columns=value.output_columns, - storage_bucket_folder_path_prefix=value.storage_bucket_folder_path_prefix, - enable_web_search_grounding=value.enable_web_search_grounding, - ) - else: - batch_transform = await uipath.context_grounding.start_batch_transform_async( - name=value.name, - index_name=value.index_name, - index_id=value.index_id, - prompt=value.prompt, - output_columns=value.output_columns, - storage_bucket_folder_path_prefix=value.storage_bucket_folder_path_prefix, - enable_web_search_grounding=value.enable_web_search_grounding, - folder_path=value.index_folder_path, - folder_key=value.index_folder_key, - ) + try: + if value.is_ephemeral_index: + batch_transform = await uipath.context_grounding.start_batch_transform_ephemeral_async( + name=value.name, + index_id=value.index_id, + prompt=value.prompt, + output_columns=value.output_columns, + storage_bucket_folder_path_prefix=value.storage_bucket_folder_path_prefix, + enable_web_search_grounding=value.enable_web_search_grounding, + ) + else: + batch_transform = await uipath.context_grounding.start_batch_transform_async( + name=value.name, + index_name=value.index_name, + index_id=value.index_id, + prompt=value.prompt, + output_columns=value.output_columns, + storage_bucket_folder_path_prefix=value.storage_bucket_folder_path_prefix, + enable_web_search_grounding=value.enable_web_search_grounding, + folder_path=value.index_folder_path, + folder_key=value.index_folder_key, + ) + except ContextGroundingIndexNotFoundError as e: + raise UiPathFaultedTriggerError( + ErrorCategory.DEPLOYMENT, + "Context grounding index not found. Check that the index is " + "deployed and available in the configured folder.", + str(e), + ) from e if not batch_transform: raise Exception("Failed to start batch transform") @@ -895,6 +984,71 @@ def _handle_api_trigger( inbox_id=str(uuid.uuid4()), request=serialize_object(value) ) + async def _handle_inbox_trigger( + self, value: WaitIntegrationEvent, resume_trigger: UiPathResumeTrigger + ) -> None: + """Handle Inbox-type resume triggers. + + Resolves `connection_name` (scoped to `connection_folder_path` when + provided) to a connection id via the Connections service, populates + `integration_resume` with the Integration Services configuration plus a + freshly generated `inbox_id`. The Connections-service registration is + performed server-side by Orchestrator's `CreateResumeTriggerTaskHandler` + once the job suspends. + + Args: + value: The suspend value (WaitIntegrationEvent) + resume_trigger: The resume trigger to populate + + Raises: + Exception: If no connection matches `connection_name`, or if more + than one exact match is found. + """ + uipath = UiPath() + connections = await uipath.connections.list_async( + name=value.connection_name, + folder_path=value.connection_folder_path, + connector_key=value.connector, + ) + connection = next( + (c for c in connections if c.name == value.connection_name), None + ) + if connection is None: + raise Exception( + f"No connection named '{value.connection_name}' " + f"for connector '{value.connector}' found" + + ( + f" in folder '{value.connection_folder_path}'" + if value.connection_folder_path + else "" + ) + ) + assert connection.id is not None + + resume_trigger.integration_resume = UiPathIntegrationTrigger( + connector=value.connector, + connection_id=connection.id, + operation=value.operation, + object_name=value.object_name, + filter_expression=value.filter_expression, + parameters=value.parameters, + inbox_id=str(uuid.uuid4()), + ) + + def _handle_time_trigger( + self, value: WaitUntil, resume_trigger: UiPathResumeTrigger + ) -> None: + """Handle Timer-type resume triggers. + + Orchestrator expects timer resume triggers as a top-level + `resumeTime` value on the resume trigger DTO. + + Args: + value: The suspend value (WaitUntil) + resume_trigger: The resume trigger to populate + """ + resume_trigger.resume_time = value.resume_time + class UiPathResumeTriggerHandler: """Combined handler for creating and reading resume triggers. @@ -921,6 +1075,10 @@ async def create_trigger(self, suspend_value: Any) -> UiPathResumeTrigger: """ return await self._creator.create_trigger(suspend_value) + async def create_triggers(self, suspend_value: Any) -> list[UiPathResumeTrigger]: + """Create resume triggers from a suspend value.""" + return await self._creator.create_triggers(suspend_value) + async def read_trigger(self, trigger: UiPathResumeTrigger) -> Any | None: """Read a resume trigger and convert it to runtime-compatible input. diff --git a/packages/uipath-platform/src/uipath/platform/semantic_proxy/__init__.py b/packages/uipath-platform/src/uipath/platform/semantic_proxy/__init__.py new file mode 100644 index 000000000..e17867ac7 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/semantic_proxy/__init__.py @@ -0,0 +1,36 @@ +"""SemanticProxy service package. + +Provides the ``SemanticProxyService`` client, Pydantic request/response models for +the PII detection endpoint, and utilities for rehydrating masked text with +original PII values after LLM processing. +""" + +from ._semantic_proxy_service import SemanticProxyService +from .pii_utilities import ( + rehydrate_from_pii_entities, + rehydrate_from_pii_response, +) +from .semantic_proxy import ( + PiiDetectionRequest, + PiiDetectionResponse, + PiiDocument, + PiiDocumentResult, + PiiEntity, + PiiEntityThreshold, + PiiFile, + PiiFileResult, +) + +__all__ = [ + "PiiDetectionRequest", + "PiiDetectionResponse", + "PiiDocument", + "PiiDocumentResult", + "PiiEntity", + "PiiEntityThreshold", + "PiiFile", + "PiiFileResult", + "SemanticProxyService", + "rehydrate_from_pii_entities", + "rehydrate_from_pii_response", +] diff --git a/packages/uipath-platform/src/uipath/platform/semantic_proxy/_semantic_proxy_service.py b/packages/uipath-platform/src/uipath/platform/semantic_proxy/_semantic_proxy_service.py new file mode 100644 index 000000000..a68d7c25c --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/semantic_proxy/_semantic_proxy_service.py @@ -0,0 +1,74 @@ +"""SemanticProxy service for UiPath Platform. + +Provides methods for interacting with the SemanticProxy service (e.g. PII detection). +""" + +from uipath.core.tracing import traced + +from ..common._base_service import BaseService +from ..common._config import UiPathApiConfig +from ..common._execution_context import UiPathExecutionContext +from ..common._models import Endpoint, RequestSpec +from .semantic_proxy import PiiDetectionRequest, PiiDetectionResponse + +_PII_DETECTION_ENDPOINT = Endpoint("semanticproxy_/api/pii-detection") + + +class SemanticProxyService(BaseService): + """Service for interacting with UiPath SemanticProxy.""" + + def __init__( + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + ) -> None: + super().__init__(config=config, execution_context=execution_context) + + @traced(name="semantic_proxy_detect_pii", run_type="uipath") + def detect_pii(self, request: PiiDetectionRequest) -> PiiDetectionResponse: + """Detect PII in the provided documents and/or files. + + Args: + request: The PII detection request payload. + + Returns: + The PII detection response. + """ + spec = self._pii_detection_spec(request) + response = self.request( + spec.method, + url=spec.endpoint, + json=spec.json, + headers=spec.headers, + scoped="tenant", + ) + return PiiDetectionResponse.model_validate(response.json()) + + @traced(name="semantic_proxy_detect_pii", run_type="uipath") + async def detect_pii_async( + self, request: PiiDetectionRequest + ) -> PiiDetectionResponse: + """Detect PII in the provided documents and/or files (async). + + Args: + request: The PII detection request payload. + + Returns: + The PII detection response. + """ + spec = self._pii_detection_spec(request) + response = await self.request_async( + spec.method, + url=spec.endpoint, + json=spec.json, + headers=spec.headers, + scoped="tenant", + ) + return PiiDetectionResponse.model_validate(response.json()) + + def _pii_detection_spec(self, request: PiiDetectionRequest) -> RequestSpec: + return RequestSpec( + method="POST", + endpoint=_PII_DETECTION_ENDPOINT, + json=request.model_dump(by_alias=True, exclude_none=True), + ) diff --git a/packages/uipath-platform/src/uipath/platform/semantic_proxy/pii_utilities.py b/packages/uipath-platform/src/uipath/platform/semantic_proxy/pii_utilities.py new file mode 100644 index 000000000..0f031a19a --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/semantic_proxy/pii_utilities.py @@ -0,0 +1,98 @@ +"""Utility methods for working with PII data. + +Python port of UiPath.SemanticProxy.Client.PiiUtilities (C#). +""" + +import re +from typing import Callable, Iterable + +from .semantic_proxy import PiiDetectionResponse, PiiEntity + + +def rehydrate_from_pii_entities( + masked_text: str, pii_entities: Iterable[PiiEntity] +) -> str: + """Rehydrate masked text by replacing PII placeholders with original values. + + Placeholders (e.g. ``[Person-1]``) are matched case-insensitively and replaced + with the corresponding original PII text. The function also replaces variants + without the surrounding brackets (e.g. ``Person-1``) in case the LLM stripped + them in its output. + + Args: + masked_text: The masked text with PII placeholders. + pii_entities: The PII entities containing the original values. + + Returns: + The rehydrated text with original PII values. + """ + if not masked_text: + return masked_text + + entities = [e for e in pii_entities if e.replacement_text] + if not entities: + return masked_text + + # Sort by replacement text length descending to avoid substring collisions + # (e.g. "[Person-10]" must be replaced before "[Person-1]"). + entities.sort(key=lambda e: len(e.replacement_text), reverse=True) + + rehydrated = masked_text + for entity in entities: + if not entity.replacement_text or not entity.pii_text: + continue + # Replace the full placeholder (with brackets) case-insensitively. + # ``_literal_replacer`` bypasses regex backreference interpretation in the + # replacement string. + rehydrated = re.sub( + re.escape(entity.replacement_text), + _literal_replacer(entity.pii_text), + rehydrated, + flags=re.IGNORECASE, + ) + # Also replace the content without brackets (in case the LLM dropped them). + if entity.replacement_text.startswith("[") and entity.replacement_text.endswith( + "]" + ): + no_brackets = entity.replacement_text[1:-1] + rehydrated = re.sub( + re.escape(no_brackets), + _literal_replacer(entity.pii_text), + rehydrated, + flags=re.IGNORECASE, + ) + + return rehydrated + + +def _literal_replacer(replacement: str) -> Callable[[re.Match[str]], str]: + """Return a replacement function that ignores regex backreference syntax.""" + + def replace(_match: re.Match[str]) -> str: + return replacement + + return replace + + +def rehydrate_from_pii_response( + masked_text: str, response: PiiDetectionResponse +) -> str: + """Rehydrate masked text using all PII entities from a detection response. + + Merges entities from both ``response.response`` (detected in documents/prompts) + and ``response.files`` (detected in files), so placeholders originating from + either source are rehydrated. + + Args: + masked_text: The masked text with PII placeholders. + response: The PII detection response containing entities to rehydrate. + + Returns: + The rehydrated text with original PII values. + """ + entities: list[PiiEntity] = [] + for doc in response.response: + entities.extend(doc.pii_entities) + for file in response.files: + entities.extend(file.pii_entities) + return rehydrate_from_pii_entities(masked_text, entities) diff --git a/packages/uipath-platform/src/uipath/platform/semantic_proxy/semantic_proxy.py b/packages/uipath-platform/src/uipath/platform/semantic_proxy/semantic_proxy.py new file mode 100644 index 000000000..2be35e975 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/semantic_proxy/semantic_proxy.py @@ -0,0 +1,91 @@ +"""Public Pydantic models for the SemanticProxy service.""" + +from typing import Optional + +from pydantic import BaseModel, ConfigDict, Field + + +class PiiDocument(BaseModel): + """A text document to scan for PII.""" + + id: str + role: str + document: str + + +class PiiFile(BaseModel): + """A file reference to scan for PII.""" + + model_config = ConfigDict(populate_by_name=True) + + file_name: str = Field(alias="fileName") + file_url: str = Field(alias="fileUrl") + file_type: str = Field(alias="fileType") + + +class PiiEntityThreshold(BaseModel): + """Per-entity confidence threshold override.""" + + model_config = ConfigDict(populate_by_name=True) + + category: str = Field(alias="pii-entity-category") + confidence_threshold: float = Field(alias="pii-entity-confidence-threshold") + + +class PiiDetectionRequest(BaseModel): + """Request payload for the PII detection endpoint.""" + + model_config = ConfigDict(populate_by_name=True) + + documents: Optional[list[PiiDocument]] = None + files: Optional[list[PiiFile]] = None + language_code: Optional[str] = Field(default=None, alias="languageCode") + confidence_threshold: Optional[float] = Field( + default=None, alias="confidenceThreshold" + ) + entity_thresholds: Optional[list[PiiEntityThreshold]] = Field( + default=None, alias="entityThresholds" + ) + + +class PiiEntity(BaseModel): + """A single detected PII entity.""" + + model_config = ConfigDict(populate_by_name=True) + + pii_text: str = Field(alias="piiText") + replacement_text: str = Field(alias="replacementText") + pii_type: str = Field(alias="piiType") + offset: int + confidence_score: float = Field(alias="confidenceScore") + + +class PiiDocumentResult(BaseModel): + """PII detection result for a single document.""" + + model_config = ConfigDict(populate_by_name=True) + + id: str + role: str + masked_document: str = Field(alias="maskedDocument") + initial_document: str = Field(alias="initialDocument") + pii_entities: list[PiiEntity] = Field(default_factory=list, alias="piiEntities") + + +class PiiFileResult(BaseModel): + """PII detection result for a single file (fileUrl is the redacted URL).""" + + model_config = ConfigDict(populate_by_name=True) + + file_name: str = Field(alias="fileName") + file_url: str = Field(alias="fileUrl") + pii_entities: list[PiiEntity] = Field(default_factory=list, alias="piiEntities") + + +class PiiDetectionResponse(BaseModel): + """Response payload from the PII detection endpoint.""" + + model_config = ConfigDict(populate_by_name=True) + + response: list[PiiDocumentResult] = Field(default_factory=list) + files: list[PiiFileResult] = Field(default_factory=list) diff --git a/packages/uipath-platform/tests/common/test_config_env_vars.py b/packages/uipath-platform/tests/common/test_config_env_vars.py new file mode 100644 index 000000000..1e48ac894 --- /dev/null +++ b/packages/uipath-platform/tests/common/test_config_env_vars.py @@ -0,0 +1,192 @@ +from pathlib import Path + +import pytest + +from uipath.platform.common._config import UiPathConfig +from uipath.platform.constants import ( + ENTRY_POINTS_FILE, + ENV_BASE_URL, + ENV_FOLDER_KEY, + ENV_FOLDER_PATH, + ENV_JOB_KEY, + ENV_ORGANIZATION_ID, + ENV_PROCESS_KEY, + ENV_PROJECT_KEY, + ENV_TENANT_ID, + ENV_TENANT_NAME, + ENV_TRACING_ENABLED, + ENV_UIPATH_AGENT_ID, + ENV_UIPATH_CLOUD_USER_ID, + ENV_UIPATH_CONFIG_PATH, + ENV_UIPATH_PROCESS_UUID, + ENV_UIPATH_PROCESS_VERSION, + ENV_UIPATH_PROJECT_FILES_SOURCE, + ENV_UIPATH_PROJECT_ID, + ENV_UIPATH_TRACE_ID, + EVALS_FOLDER, + LEGACY_EVAL_FOLDER, + STUDIO_METADATA_FILE, + UIPATH_BINDINGS_FILE, + UIPATH_CONFIG_FILE, + UIPROJ_FILE, +) + +# Every env var read by ConfigurationManager properties. Cleared before each +# test so "returns None when unset" assertions don't pick up the real +# environment. +_ENV_VARS = ( + ENV_UIPATH_PROJECT_ID, + ENV_UIPATH_AGENT_ID, + ENV_UIPATH_CLOUD_USER_ID, + ENV_UIPATH_PROJECT_FILES_SOURCE, + ENV_PROJECT_KEY, + ENV_TENANT_NAME, + ENV_TENANT_ID, + ENV_ORGANIZATION_ID, + ENV_BASE_URL, + ENV_FOLDER_KEY, + ENV_FOLDER_PATH, + ENV_PROCESS_KEY, + ENV_UIPATH_PROCESS_UUID, + ENV_UIPATH_TRACE_ID, + ENV_UIPATH_PROCESS_VERSION, + ENV_JOB_KEY, + ENV_UIPATH_CONFIG_PATH, + ENV_TRACING_ENABLED, +) + + +@pytest.fixture(autouse=True) +def _clear_env(monkeypatch): + for var in _ENV_VARS: + monkeypatch.delenv(var, raising=False) + + +class TestProjectId: + def test_reads_env_var(self, monkeypatch): + monkeypatch.setenv(ENV_UIPATH_PROJECT_ID, "file-source-id") + assert UiPathConfig.project_id == "file-source-id" + + def test_returns_none_when_unset(self): + assert UiPathConfig.project_id is None + + +class TestAgentId: + def test_returns_explicit_agent_id_when_set(self, monkeypatch): + monkeypatch.setenv(ENV_UIPATH_PROJECT_ID, "debug-project-guid") + monkeypatch.setenv(ENV_UIPATH_AGENT_ID, "real-agent-id") + assert UiPathConfig.agent_id == "real-agent-id" + + def test_falls_back_to_project_id_when_agent_id_unset(self, monkeypatch): + monkeypatch.setenv(ENV_UIPATH_PROJECT_ID, "cloud-project-id") + assert UiPathConfig.agent_id == "cloud-project-id" + + def test_returns_none_when_neither_set(self): + assert UiPathConfig.agent_id is None + + +class TestCloudUserId: + def test_returns_value_when_set(self, monkeypatch): + monkeypatch.setenv(ENV_UIPATH_CLOUD_USER_ID, "user-guid") + assert UiPathConfig.cloud_user_id == "user-guid" + + def test_returns_none_when_unset(self): + assert UiPathConfig.cloud_user_id is None + + +class TestProjectFilesSource: + def test_returns_value_when_set(self, monkeypatch): + monkeypatch.setenv(ENV_UIPATH_PROJECT_FILES_SOURCE, "Local") + assert UiPathConfig.project_files_source == "Local" + + def test_returns_none_when_unset(self): + assert UiPathConfig.project_files_source is None + + +@pytest.mark.parametrize( + ("prop", "env_var", "value"), + [ + ("project_key", ENV_PROJECT_KEY, "proj-key"), + ("tenant_name", ENV_TENANT_NAME, "my-tenant"), + ("tenant_id", ENV_TENANT_ID, "tenant-guid"), + ("organization_id", ENV_ORGANIZATION_ID, "org-guid"), + ("base_url", ENV_BASE_URL, "https://cloud.uipath.com/org/tenant"), + ("folder_key", ENV_FOLDER_KEY, "folder-guid"), + ("folder_path", ENV_FOLDER_PATH, "Shared/My Folder"), + ("process_key", ENV_PROCESS_KEY, "process-key"), + ("process_uuid", ENV_UIPATH_PROCESS_UUID, "process-uuid"), + ("trace_id", ENV_UIPATH_TRACE_ID, "trace-id"), + ("process_version", ENV_UIPATH_PROCESS_VERSION, "1.2.3"), + ("job_key", ENV_JOB_KEY, "job-guid"), + ], +) +class TestOptionalEnvVarProperties: + def test_returns_value_when_set(self, monkeypatch, prop, env_var, value): + monkeypatch.setenv(env_var, value) + assert getattr(UiPathConfig, prop) == value + + def test_returns_none_when_unset(self, monkeypatch, prop, env_var, value): + assert getattr(UiPathConfig, prop) is None + + +class TestFileNameProperties: + def test_config_file_name(self): + assert UiPathConfig.config_file_name == UIPATH_CONFIG_FILE + + def test_bindings_file_path(self): + assert UiPathConfig.bindings_file_path == Path(UIPATH_BINDINGS_FILE) + + def test_entry_points_file_path(self): + assert UiPathConfig.entry_points_file_path == Path(ENTRY_POINTS_FILE) + + def test_uiproj_file_path(self): + assert UiPathConfig.uiproj_file_path == Path(UIPROJ_FILE) + + def test_studio_metadata_file_path(self): + assert UiPathConfig.studio_metadata_file_path == Path( + ".uipath", STUDIO_METADATA_FILE + ) + + def test_config_file_path_defaults_to_config_file(self): + assert UiPathConfig.config_file_path == Path(UIPATH_CONFIG_FILE) + + def test_config_file_path_honors_override(self, monkeypatch): + monkeypatch.setenv(ENV_UIPATH_CONFIG_PATH, "custom/config.json") + assert UiPathConfig.config_file_path == Path("custom/config.json") + + +class TestIsStudioProject: + def test_true_when_project_id_set(self, monkeypatch): + monkeypatch.setenv(ENV_UIPATH_PROJECT_ID, "some-id") + assert UiPathConfig.is_studio_project is True + + def test_false_when_project_id_unset(self): + assert UiPathConfig.is_studio_project is False + + +class TestIsTracingEnabled: + def test_defaults_to_true_when_unset(self): + assert UiPathConfig.is_tracing_enabled is True + + @pytest.mark.parametrize("value", ["false", "False", "FALSE"]) + def test_false_when_disabled(self, monkeypatch, value): + monkeypatch.setenv(ENV_TRACING_ENABLED, value) + assert UiPathConfig.is_tracing_enabled is False + + def test_true_when_explicitly_enabled(self, monkeypatch): + monkeypatch.setenv(ENV_TRACING_ENABLED, "true") + assert UiPathConfig.is_tracing_enabled is True + + +class TestEvalFolderDetection: + def test_has_legacy_eval_folder(self, monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + assert UiPathConfig.has_legacy_eval_folder is False + (tmp_path / LEGACY_EVAL_FOLDER).mkdir() + assert UiPathConfig.has_legacy_eval_folder is True + + def test_has_eval_folder(self, monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + assert UiPathConfig.has_eval_folder is False + (tmp_path / EVALS_FOLDER).mkdir() + assert UiPathConfig.has_eval_folder is True diff --git a/packages/uipath-platform/tests/common/test_constants_invariants.py b/packages/uipath-platform/tests/common/test_constants_invariants.py new file mode 100644 index 000000000..f85b36b78 --- /dev/null +++ b/packages/uipath-platform/tests/common/test_constants_invariants.py @@ -0,0 +1,25 @@ +"""Structural invariants for the canonical constants module. + +Exercises ``uipath.platform.constants`` (the source of truth) directly. +Asserts structural contracts, not literal values — a literal-mirror test would +just restate the module. +""" + +import uipath.platform.constants as constants + + +def _public_constants() -> dict[str, object]: + return {n: getattr(constants, n) for n in dir(constants) if not n.startswith("_")} + + +def test_all_constants_are_non_empty_strings(): + for name, value in _public_constants().items(): + assert isinstance(value, str) and value, ( + f"{name} must be a non-empty string, got {value!r}" + ) + + +def test_no_duplicate_header_values(): + # Two header constants mapping to the same wire name is almost certainly a bug. + headers = [v for n, v in _public_constants().items() if n.startswith("HEADER_")] + assert len(headers) == len(set(headers)), "duplicate header wire names detected" diff --git a/packages/uipath-platform/tests/common/test_constants_reexport.py b/packages/uipath-platform/tests/common/test_constants_reexport.py new file mode 100644 index 000000000..484c87731 --- /dev/null +++ b/packages/uipath-platform/tests/common/test_constants_reexport.py @@ -0,0 +1,67 @@ +"""Tests for the constants source of truth and its deprecated re-export. + +``uipath.platform.constants`` is the single source of truth. +``uipath.platform.common.constants`` is a deprecated shim that re-exports it +and emits a FutureWarning. These tests pin the parity invariant (so the two can +never drift — same name, different value) and the deprecation behavior. +""" + +import importlib +import sys +import warnings + + +def _public_names(module) -> set[str]: + return {name for name in dir(module) if not name.startswith("_")} + + +def _import_shim(): + """Import the deprecated shim with its FutureWarning suppressed.""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + return importlib.import_module("uipath.platform.common.constants") + + +def test_shim_reexports_canonical_with_no_drift(): + canonical = importlib.import_module("uipath.platform.constants") + shim = _import_shim() + + canonical_names = _public_names(canonical) + + # Every canonical constant is re-exported with an identical value. + missing = sorted(n for n in canonical_names if not hasattr(shim, n)) + assert not missing, f"shim missing constants: {missing}" + + drift = sorted( + n for n in canonical_names if getattr(shim, n) != getattr(canonical, n) + ) + assert not drift, f"value drift between shim and canonical for: {drift}" + + +def test_shim_adds_nothing_of_its_own(): + """The shim must not define constants absent from the canonical module, + otherwise it would no longer be the single source of truth.""" + canonical = importlib.import_module("uipath.platform.constants") + shim = _import_shim() + + extra = sorted(_public_names(shim) - _public_names(canonical)) + assert not extra, f"shim defines names not in canonical module: {extra}" + + +def test_shim_emits_future_warning(): + sys.modules.pop("uipath.platform.common.constants", None) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + importlib.import_module("uipath.platform.common.constants") + + shim_warnings = [ + w + for w in caught + if issubclass(w.category, FutureWarning) + and "uipath.platform.common.constants" in str(w.message) + and "uipath.platform.constants" in str(w.message) + ] + assert len(shim_warnings) == 1, ( + f"expected exactly one deprecation FutureWarning, got {len(shim_warnings)}: " + f"{[str(w.message) for w in caught]}" + ) diff --git a/packages/uipath-platform/tests/common/test_execution_context.py b/packages/uipath-platform/tests/common/test_execution_context.py new file mode 100644 index 000000000..25b43c9b2 --- /dev/null +++ b/packages/uipath-platform/tests/common/test_execution_context.py @@ -0,0 +1,26 @@ +from uipath.platform.common import ExecutionSourceContext, UiPathExecutionContext + + +def test_execution_source_none_by_default() -> None: + assert UiPathExecutionContext().execution_source is None + + +def test_execution_source_set_within_context() -> None: + ctx = UiPathExecutionContext() + + with ExecutionSourceContext("runtime"): + assert ctx.execution_source == "runtime" + + assert ctx.execution_source is None + + +def test_execution_source_context_restores_previous_value() -> None: + ctx = UiPathExecutionContext() + + with ExecutionSourceContext("eval"): + assert ctx.execution_source == "eval" + with ExecutionSourceContext("playground"): + assert ctx.execution_source == "playground" + assert ctx.execution_source == "eval" + + assert ctx.execution_source is None diff --git a/packages/uipath-platform/tests/common/test_job_context.py b/packages/uipath-platform/tests/common/test_job_context.py new file mode 100644 index 000000000..48e603d7b --- /dev/null +++ b/packages/uipath-platform/tests/common/test_job_context.py @@ -0,0 +1,22 @@ +import pytest + +from uipath.platform.common._job_context import header_job_key +from uipath.platform.constants import ENV_JOB_KEY, HEADER_JOB_KEY + + +def test_returns_header_when_env_var_set(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(ENV_JOB_KEY, "test-job-key") + + assert header_job_key() == {HEADER_JOB_KEY: "test-job-key"} + + +def test_returns_empty_when_env_var_unset(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv(ENV_JOB_KEY, raising=False) + + assert header_job_key() == {} + + +def test_returns_empty_when_env_var_blank(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(ENV_JOB_KEY, "") + + assert header_job_key() == {} diff --git a/packages/uipath-platform/tests/common/test_timeout_helpers.py b/packages/uipath-platform/tests/common/test_timeout_helpers.py new file mode 100644 index 000000000..a29f1524d --- /dev/null +++ b/packages/uipath-platform/tests/common/test_timeout_helpers.py @@ -0,0 +1,96 @@ +import pytest +from uipath.core.triggers import ( + UIPATH_METADATA_KEY, + UiPathResumeTriggerName, + UiPathResumeTriggerType, +) + +from uipath.platform.common import ( + UiPathResumeMetadata, + UiPathTimeoutError, + assert_no_timeout, + get_resume_metadata, + is_timeout, +) + + +def test_is_timeout_detects_timeout_metadata() -> None: + value = { + UIPATH_METADATA_KEY: { + "triggerType": "Timer", + "triggerName": "Timer", + }, + "value": None, + } + + assert is_timeout(value) is True + + +def test_get_resume_metadata_returns_typed_metadata() -> None: + value = { + UIPATH_METADATA_KEY: { + "triggerType": "Timer", + "triggerName": "Timer", + }, + "resumeTime": "2026-07-07T12:00:00Z", + } + + metadata = get_resume_metadata(value) + + assert isinstance(metadata, UiPathResumeMetadata) + assert metadata.trigger_type == UiPathResumeTriggerType.TIMER + assert metadata.trigger_name == UiPathResumeTriggerName.TIMER + + +def test_get_resume_metadata_returns_none_without_metadata() -> None: + assert get_resume_metadata({"result": "done"}) is None + assert get_resume_metadata("done") is None + + +def test_get_resume_metadata_returns_none_for_invalid_metadata() -> None: + value = { + UIPATH_METADATA_KEY: { + "triggerType": "NotARealTrigger", + "triggerName": "Timer", + }, + } + + assert get_resume_metadata(value) is None + + +def test_is_timeout_ignores_non_timer_metadata() -> None: + value = { + UIPATH_METADATA_KEY: { + "triggerType": "Job", + "triggerName": "Job", + }, + "value": {"jobKey": "job-1"}, + } + + assert is_timeout(value) is False + + +def test_is_timeout_ignores_user_timed_out_fields() -> None: + assert is_timeout({"timedOut": True}) is False + assert is_timeout("timeout") is False + + +def test_assert_no_timeout_returns_original_value() -> None: + value = {"result": "done"} + + assert assert_no_timeout(value) is value + + +def test_assert_no_timeout_raises_with_resume_value() -> None: + value = { + UIPATH_METADATA_KEY: { + "triggerType": "Timer", + "triggerName": "Timer", + }, + "value": {"jobKey": "job-1"}, + } + + with pytest.raises(UiPathTimeoutError) as exc_info: + assert_no_timeout(value) + + assert exc_info.value.value is value diff --git a/packages/uipath-platform/tests/errors/test_datafabric_errors.py b/packages/uipath-platform/tests/errors/test_datafabric_errors.py new file mode 100644 index 000000000..39c477ab5 --- /dev/null +++ b/packages/uipath-platform/tests/errors/test_datafabric_errors.py @@ -0,0 +1,189 @@ +"""Tests for Data Fabric error classification, extraction, and routing.""" + +import json + +import httpx + +from uipath.platform.errors import ( + DataFabricError, + DataFabricErrorCategory, + EnrichedException, +) +from uipath.platform.errors._extractors._datafabric import extract_datafabric +from uipath.platform.errors._extractors._router import extract_error_info +from uipath.platform.errors.datafabric_error_codes import classify_error_code + +_DATAFABRIC_URL = "https://cloud.uipath.com/org/tenant/datafabric_/api/v1" +_NON_DF_URL = "https://cloud.uipath.com/org/tenant/orchestrator_/api/v1" + + +def _make_enriched( + url: str = _DATAFABRIC_URL, + body: str = "{}", + status_code: int = 400, +) -> EnrichedException: + raw = httpx.HTTPStatusError( + message=f"Server error {status_code}", + request=httpx.Request("POST", url), + response=httpx.Response( + status_code, + content=body.encode(), + headers={"content-type": "application/json"}, + ), + ) + return EnrichedException(raw) + + +# ---------- classify_error_code ---------- + + +class TestClassifyErrorCode: + def test_retryable_codes(self) -> None: + for code in ("EXECUTION_TIMEOUT", "SQLITE_BUSY", "EXECUTION_INTERRUPTED"): + assert classify_error_code(code) == DataFabricErrorCategory.RETRYABLE + + def test_bad_sql_codes(self) -> None: + for code in ("SQL_PARSING", "SQL_VALIDATION"): + assert classify_error_code(code) == DataFabricErrorCategory.BAD_SQL + + def test_infrastructure_codes(self) -> None: + for code in ( + "SQLITE_MEMORY_FULL", + "EPHEMERAL_STORAGE_ERROR", + "INTERNAL_ERROR", + "FQS_ERROR", + ): + assert classify_error_code(code) == DataFabricErrorCategory.INFRASTRUCTURE + + def test_data_issue_codes(self) -> None: + for code in ( + "FRAGMENT_EXECUTION_FAILURE", + "CONTEXT_CREATION", + "UNKNOWN_ENTITY", + "EXECUTION_ERROR", + "RESULT_TOO_LARGE", + ): + assert classify_error_code(code) == DataFabricErrorCategory.DATA_ISSUE + + def test_unknown_code(self) -> None: + assert classify_error_code("NEVER_HEARD_OF") == DataFabricErrorCategory.UNKNOWN + + def test_none_code(self) -> None: + assert classify_error_code(None) == DataFabricErrorCategory.UNKNOWN + + def test_empty_string(self) -> None: + assert classify_error_code("") == DataFabricErrorCategory.UNKNOWN + + def test_case_insensitive(self) -> None: + assert classify_error_code("sql_parsing") == DataFabricErrorCategory.BAD_SQL + assert ( + classify_error_code("Execution_Timeout") + == DataFabricErrorCategory.RETRYABLE + ) + + +# ---------- DataFabricError ---------- + + +class TestDataFabricError: + def test_is_retryable(self) -> None: + err = DataFabricError( + code="EXECUTION_TIMEOUT", + message="timed out", + trace_id="abc", + category=DataFabricErrorCategory.RETRYABLE, + ) + assert err.is_retryable is True + assert err.is_bad_sql is False + + def test_is_bad_sql(self) -> None: + err = DataFabricError( + code="SQL_PARSING", + message="bad sql", + trace_id="abc", + category=DataFabricErrorCategory.BAD_SQL, + ) + assert err.is_bad_sql is True + assert err.is_retryable is False + + def test_from_response_body(self) -> None: + body = { + "error": "something went wrong", + "code": "INTERNAL_ERROR", + "traceId": "trace-123", + } + err = DataFabricError.from_response_body(body) + assert err.code == "INTERNAL_ERROR" + assert err.message == "something went wrong" + assert err.trace_id == "trace-123" + assert err.category == DataFabricErrorCategory.INFRASTRUCTURE + + def test_from_response_body_missing_fields(self) -> None: + err = DataFabricError.from_response_body({}) + assert err.code is None + assert err.message is None + assert err.trace_id is None + assert err.category == DataFabricErrorCategory.UNKNOWN + + def test_from_enriched_exception_datafabric_url(self) -> None: + body = json.dumps({"error": "bad sql", "code": "SQL_PARSING", "traceId": "t-1"}) + exc = _make_enriched(url=_DATAFABRIC_URL, body=body) + err = DataFabricError.from_enriched_exception(exc) + assert err is not None + assert err.code == "SQL_PARSING" + assert err.message == "bad sql" + assert err.trace_id == "t-1" + assert err.category == DataFabricErrorCategory.BAD_SQL + + def test_from_enriched_exception_non_datafabric_url_returns_none(self) -> None: + body = json.dumps({"error": "oops", "code": "SQL_PARSING"}) + exc = _make_enriched(url=_NON_DF_URL, body=body) + assert DataFabricError.from_enriched_exception(exc) is None + + def test_from_enriched_exception_no_error_info(self) -> None: + exc = _make_enriched(url=_DATAFABRIC_URL, body="not json at all {{{") + err = DataFabricError.from_enriched_exception(exc) + assert err is not None + assert err.code is None + assert err.message is None + assert err.category == DataFabricErrorCategory.UNKNOWN + + +# ---------- extract_datafabric ---------- + + +class TestExtractDatafabric: + def test_extracts_all_fields(self) -> None: + body = {"error": "msg", "code": "SQL_PARSING", "traceId": "t-1"} + info = extract_datafabric(body) + assert info.message == "msg" + assert info.error_code == "SQL_PARSING" + assert info.trace_id == "t-1" + + def test_falls_back_to_message_key(self) -> None: + body = {"message": "fallback msg", "code": "X"} + info = extract_datafabric(body) + assert info.message == "fallback msg" + + def test_missing_fields(self) -> None: + info = extract_datafabric({}) + assert info.message is None + assert info.error_code is None + assert info.trace_id is None + + +# ---------- Router: datafabric prefix ---------- + + +class TestRouterDatafabric: + def test_routes_to_datafabric_extractor(self) -> None: + body = json.dumps( + {"error": "timeout", "code": "EXECUTION_TIMEOUT", "traceId": "t-2"} + ) + info = extract_error_info(_DATAFABRIC_URL, body) + assert info is not None + assert info.error_code == "EXECUTION_TIMEOUT" + assert info.trace_id == "t-2" + + def test_non_json_returns_none(self) -> None: + assert extract_error_info(_DATAFABRIC_URL, "not json") is None diff --git a/packages/uipath-platform/tests/services/test_actions_service.py b/packages/uipath-platform/tests/services/test_actions_service.py index b97d326e8..28180dbbb 100644 --- a/packages/uipath-platform/tests/services/test_actions_service.py +++ b/packages/uipath-platform/tests/services/test_actions_service.py @@ -1,3 +1,4 @@ +import json from typing import Any import pytest @@ -6,7 +7,8 @@ from uipath.platform import UiPathApiConfig, UiPathExecutionContext from uipath.platform.action_center import Task from uipath.platform.action_center._tasks_service import TasksService -from uipath.platform.common.constants import HEADER_USER_AGENT +from uipath.platform.action_center.tasks import TaskRecipient, TaskRecipientType +from uipath.platform.constants import HEADER_USER_AGENT @pytest.fixture @@ -185,6 +187,167 @@ def test_create_with_assignee( assert action.title == "Test Action" +def _mock_app_lookup_and_create( + httpx_mock: HTTPXMock, + base_url: str, + org: str, + tenant: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Common httpx mock setup for app lookup + task creation + assign.""" + monkeypatch.setenv("UIPATH_TENANT_ID", "test-tenant-id") + httpx_mock.add_response( + url=f"{base_url}{org}/apps_/default/api/v1/default/deployed-action-apps-schemas?search=test-app&filterByDeploymentTitle=true", + status_code=200, + json={ + "deployed": [ + { + "systemName": "test-app", + "deploymentTitle": "test-app", + "actionSchema": { + "key": "test-key", + "inputs": [], + "outputs": [], + "inOuts": [], + "outcomes": [], + }, + "deploymentFolder": { + "fullyQualifiedName": "test-folder-path", + "key": "test-folder-key", + }, + } + ] + }, + ) + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/tasks/AppTasks/CreateAppTask", + status_code=200, + json={"id": 1, "title": "Test Action"}, + ) + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/odata/Tasks/UiPath.Server.Configuration.OData.AssignTasks", + status_code=200, + json={}, + ) + + +def _assign_request_payload(httpx_mock: HTTPXMock) -> dict[str, Any]: + """Return the parsed JSON body of the last AssignTasks request captured by the mock.""" + assign_request = next( + req + for req in reversed(httpx_mock.get_requests()) + if "AssignTasks" in str(req.url) + ) + return json.loads(assign_request.content) + + +class TestAssignTaskSpec: + """Tests for the task-assignment payload built by `_assign_task_spec`.""" + + def test_assign_workload_recipient_uses_workload_criteria_with_group( + self, + httpx_mock: HTTPXMock, + service: TasksService, + base_url: str, + org: str, + tenant: str, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + _mock_app_lookup_and_create(httpx_mock, base_url, org, tenant, monkeypatch) + + service.create( + title="Test Action", + app_name="test-app", + data={"x": 1}, + recipient=TaskRecipient( + type=TaskRecipientType.WORKLOAD, + value="Support Team", + displayName="Support Team", + ), + ) + + payload = _assign_request_payload(httpx_mock) + assert payload == { + "taskAssignments": [ + { + "taskId": 1, + "assignmentCriteria": "Workload", + "assigneeNamesOrEmails": ["Support Team"], + } + ] + } + + def test_assign_round_robin_recipient_uses_round_robin_criteria( + self, + httpx_mock: HTTPXMock, + service: TasksService, + base_url: str, + org: str, + tenant: str, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + _mock_app_lookup_and_create(httpx_mock, base_url, org, tenant, monkeypatch) + + service.create( + title="Test Action", + app_name="test-app", + data={"x": 1}, + recipient=TaskRecipient( + type=TaskRecipientType.ROUND_ROBIN, + value="Support Team", + displayName="Support Team", + ), + ) + + payload = _assign_request_payload(httpx_mock) + assert payload == { + "taskAssignments": [ + { + "taskId": 1, + "assignmentCriteria": "RoundRobin", + "assigneeNamesOrEmails": ["Support Team"], + } + ] + } + + def test_assign_workload_with_multiple_emails_uses_values_list( + self, + httpx_mock: HTTPXMock, + service: TasksService, + base_url: str, + org: str, + tenant: str, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Custom-assignees path: Workload criteria with a list of emails.""" + _mock_app_lookup_and_create(httpx_mock, base_url, org, tenant, monkeypatch) + + service.create( + title="Test Action", + app_name="test-app", + data={"x": 1}, + recipient=TaskRecipient( + type=TaskRecipientType.WORKLOAD, + value="alice@example.com", + values=["alice@example.com", "bob@example.com"], + ), + ) + + payload = _assign_request_payload(httpx_mock) + assert payload == { + "taskAssignments": [ + { + "taskId": 1, + "assignmentCriteria": "Workload", + "assigneeNamesOrEmails": [ + "alice@example.com", + "bob@example.com", + ], + } + ] + } + + def _make_deployed_app( name: str, folder_path: str, @@ -555,3 +718,147 @@ def test_create_raises_when_no_folder_key_or_path_provided( app_name="my-app", app_folder_path=None, ) + + +# --------------------------------------------------------------------------- +# QuickForm task tests +# --------------------------------------------------------------------------- + +_QF_SCHEMA: dict[str, Any] = { + "id": "7ebef452-fee9-45df-8fc2-01f1d0248540", + "fields": [ + {"id": "f1", "type": "text", "label": "F1", "direction": "input"}, + {"id": "f2", "type": "text", "label": "F2", "direction": "output"}, + ], + "outcomes": [ + {"id": "approve", "name": "Approve", "type": "string", "isPrimary": True}, + ], +} +_QF_DEFAULTS = { + "title": "QF task", + "task_schema_key": _QF_SCHEMA["id"], + "schema": _QF_SCHEMA, +} +_QF_CREATE_RESPONSE = {"id": 42, "title": _QF_DEFAULTS["title"]} + + +@pytest.fixture +def qf_create_url(base_url: str, org: str, tenant: str) -> str: + return f"{base_url}{org}{tenant}/orchestrator_/tasks/GenericTasks/CreateTask" + + +@pytest.fixture +def qf_assign_url(base_url: str, org: str, tenant: str) -> str: + return ( + f"{base_url}{org}{tenant}" + "/orchestrator_/odata/Tasks/UiPath.Server.Configuration.OData.AssignTasks" + ) + + +def _posted_body(httpx_mock: HTTPXMock, url: str) -> dict[str, Any]: + for req in httpx_mock.get_requests(): + if str(req.url) == url: + return json.loads(req.content) + raise AssertionError(f"no request was POSTed to {url}") + + +@pytest.fixture +def qf_runner(httpx_mock: HTTPXMock, service: TasksService, qf_create_url: str) -> Any: + """Factory: stub the QF endpoint, call create_quickform with overrides, + return (task, posted_body). One call per test eliminates setup duplication. + """ + httpx_mock.add_response( + url=qf_create_url, status_code=200, json=_QF_CREATE_RESPONSE + ) + + def _run(**overrides: Any) -> tuple[Task, dict[str, Any]]: + task = service.create_quickform(**{**_QF_DEFAULTS, **overrides}) + return task, _posted_body(httpx_mock, qf_create_url) + + return _run + + +@pytest.fixture +def qf_runner_async( + httpx_mock: HTTPXMock, service: TasksService, qf_create_url: str +) -> Any: + """Async variant of qf_runner.""" + httpx_mock.add_response( + url=qf_create_url, status_code=200, json=_QF_CREATE_RESPONSE + ) + + async def _run(**overrides: Any) -> tuple[Task, dict[str, Any]]: + task = await service.create_quickform_async(**{**_QF_DEFAULTS, **overrides}) + return task, _posted_body(httpx_mock, qf_create_url) + + return _run + + +def test_create_quickform_baseline_payload(qf_runner: Any) -> None: + task, body = qf_runner() + assert body == { + "type": 6, + "taskSchemaKey": _QF_DEFAULTS["task_schema_key"], + "schema": _QF_SCHEMA, + "title": _QF_DEFAULTS["title"], + "data": {}, + } + assert isinstance(task, Task) + assert task.id == 42 + + +def test_create_quickform_data_passthrough(qf_runner: Any) -> None: + _, body = qf_runner(data={"x": 1}) + assert body["data"] == {"x": 1} + + +def test_create_quickform_includes_optional_fields_when_set(qf_runner: Any) -> None: + _, body = qf_runner( + priority="High", + labels=["a", "b"], + is_actionable_message_enabled=True, + actionable_message_metadata={"fieldSet": {}, "actionSet": {}}, + creator_job_key="3fa85f64-5717-4562-b3fc-2c963f66afa6", + ) + assert body["priority"] == "High" + assert {tag["name"] for tag in body["tags"]} == {"a", "b"} + assert body["isActionableMessageEnabled"] is True + assert body["actionableMessageMetaData"] == {"fieldSet": {}, "actionSet": {}} + assert body["creatorJobKey"] == "3fa85f64-5717-4562-b3fc-2c963f66afa6" + + +def test_create_quickform_omits_optional_fields_when_unset(qf_runner: Any) -> None: + _, body = qf_runner() + for omitted in ( + "creatorJobKey", + "priority", + "tags", + "isActionableMessageEnabled", + "actionableMessageMetaData", + ): + assert omitted not in body + + +async def test_create_quickform_async_baseline_payload(qf_runner_async: Any) -> None: + task, body = await qf_runner_async() + assert body["type"] == 6 + assert body["taskSchemaKey"] == _QF_DEFAULTS["task_schema_key"] + assert task.id == 42 + + +def test_create_quickform_with_assignee_triggers_assign_call( + httpx_mock: HTTPXMock, qf_runner: Any, qf_assign_url: str +) -> None: + httpx_mock.add_response(url=qf_assign_url, status_code=200, json={}) + qf_runner(assignee="user@example.com") + body = _posted_body(httpx_mock, qf_assign_url) + assert body["taskAssignments"][0]["UserNameOrEmail"] == "user@example.com" + + +async def test_create_quickform_async_with_assignee_triggers_assign_call( + httpx_mock: HTTPXMock, qf_runner_async: Any, qf_assign_url: str +) -> None: + httpx_mock.add_response(url=qf_assign_url, status_code=200, json={}) + await qf_runner_async(assignee="user@example.com") + body = _posted_body(httpx_mock, qf_assign_url) + assert body["taskAssignments"][0]["UserNameOrEmail"] == "user@example.com" diff --git a/packages/uipath-platform/tests/services/test_api_client.py b/packages/uipath-platform/tests/services/test_api_client.py index eac74fa3c..842b149ff 100644 --- a/packages/uipath-platform/tests/services/test_api_client.py +++ b/packages/uipath-platform/tests/services/test_api_client.py @@ -3,7 +3,7 @@ from uipath.platform import UiPathApiConfig, UiPathExecutionContext from uipath.platform.common._api_client import ApiClient -from uipath.platform.common.constants import HEADER_USER_AGENT +from uipath.platform.constants import HEADER_USER_AGENT @pytest.fixture diff --git a/packages/uipath-platform/tests/services/test_assets_service.py b/packages/uipath-platform/tests/services/test_assets_service.py index 6e83c3b9d..94f8ab888 100644 --- a/packages/uipath-platform/tests/services/test_assets_service.py +++ b/packages/uipath-platform/tests/services/test_assets_service.py @@ -4,8 +4,8 @@ from pytest_httpx import HTTPXMock from uipath.platform import UiPathApiConfig, UiPathExecutionContext -from uipath.platform.common.constants import HEADER_USER_AGENT from uipath.platform.common.paging import PagedResult +from uipath.platform.constants import HEADER_USER_AGENT from uipath.platform.orchestrator import Asset, UserAsset from uipath.platform.orchestrator._assets_service import AssetsService @@ -362,20 +362,94 @@ def test_retrieve_credential( == f"UiPath.Python.Sdk/UiPath.Python.Sdk.Activities.AssetsService.retrieve_credential/{version}" ) - def test_retrieve_credential_user_asset( + def test_retrieve_credential_no_robot_key_direct_access_disabled( self, - service: AssetsService, - monkeypatch: pytest.MonkeyPatch, + httpx_mock: HTTPXMock, + base_url: str, + org: str, + tenant: str, config: UiPathApiConfig, + monkeypatch: pytest.MonkeyPatch, ) -> None: + monkeypatch.delenv("UIPATH_ROBOT_KEY", raising=False) + service = AssetsService( + config=config, + execution_context=UiPathExecutionContext(), + ) + + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/odata/Assets/UiPath.Server.Configuration.OData.GetFiltered?$filter=Name eq 'Test Credential'&$top=1", + status_code=200, + json={ + "value": [ + { + "Key": "asset-key", + "Name": "Test Credential", + "ValueType": "Credential", + "AllowDirectApiAccess": False, + } + ] + }, + ) + with pytest.raises(ValueError): - monkeypatch.delenv("UIPATH_ROBOT_KEY", raising=False) - service = AssetsService( - config=config, - execution_context=UiPathExecutionContext(), - ) service.retrieve_credential(name="Test Credential") + def test_retrieve_credential_no_robot_key_direct_access_enabled( + self, + httpx_mock: HTTPXMock, + base_url: str, + org: str, + tenant: str, + config: UiPathApiConfig, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + import json + + monkeypatch.delenv("UIPATH_ROBOT_KEY", raising=False) + service = AssetsService( + config=config, + execution_context=UiPathExecutionContext(), + ) + + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/odata/Assets/UiPath.Server.Configuration.OData.GetFiltered?$filter=Name eq 'Test Credential'&$top=1", + status_code=200, + json={ + "value": [ + { + "Key": "asset-key", + "Name": "Test Credential", + "ValueType": "Credential", + "AllowDirectApiAccess": True, + } + ] + }, + ) + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/odata/Assets/UiPath.Server.Configuration.OData.GetRobotAssetByNameForRobotKey", + status_code=200, + json={ + "id": 1, + "name": "Test Credential", + "credential_username": "test-user", + "credential_password": "test-password", + }, + ) + + credential = service.retrieve_credential(name="Test Credential") + + assert credential == "test-password" + + sent_requests = httpx_mock.get_requests() + assert len(sent_requests) == 2 + credential_request = sent_requests[1] + assert credential_request.method == "POST" + request_body = json.loads(credential_request.content) + assert request_body["assetName"] == "Test Credential" + assert request_body["supportsCredentialsProxyDisconnected"] is True + assert "robotKey" not in request_body + async def test_retrieve_credential_async( self, httpx_mock: HTTPXMock, @@ -417,6 +491,196 @@ async def test_retrieve_credential_async( == f"UiPath.Python.Sdk/UiPath.Python.Sdk.Activities.AssetsService.retrieve_credential_async/{version}" ) + @pytest.mark.anyio + async def test_retrieve_credential_async_no_robot_key_direct_access_disabled( + self, + httpx_mock: HTTPXMock, + base_url: str, + org: str, + tenant: str, + config: UiPathApiConfig, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.delenv("UIPATH_ROBOT_KEY", raising=False) + service = AssetsService( + config=config, + execution_context=UiPathExecutionContext(), + ) + + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/odata/Assets/UiPath.Server.Configuration.OData.GetFiltered?$filter=Name eq 'Test Credential'&$top=1", + status_code=200, + json={ + "value": [ + { + "Key": "asset-key", + "Name": "Test Credential", + "ValueType": "Credential", + "AllowDirectApiAccess": False, + } + ] + }, + ) + + with pytest.raises(ValueError): + await service.retrieve_credential_async(name="Test Credential") + + @pytest.mark.anyio + async def test_retrieve_credential_async_no_robot_key_direct_access_enabled( + self, + httpx_mock: HTTPXMock, + base_url: str, + org: str, + tenant: str, + config: UiPathApiConfig, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + import json + + monkeypatch.delenv("UIPATH_ROBOT_KEY", raising=False) + service = AssetsService( + config=config, + execution_context=UiPathExecutionContext(), + ) + + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/odata/Assets/UiPath.Server.Configuration.OData.GetFiltered?$filter=Name eq 'Test Credential'&$top=1", + status_code=200, + json={ + "value": [ + { + "Key": "asset-key", + "Name": "Test Credential", + "ValueType": "Credential", + "AllowDirectApiAccess": True, + } + ] + }, + ) + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/odata/Assets/UiPath.Server.Configuration.OData.GetRobotAssetByNameForRobotKey", + status_code=200, + json={ + "id": 1, + "name": "Test Credential", + "credential_username": "test-user", + "credential_password": "test-password", + }, + ) + + credential = await service.retrieve_credential_async(name="Test Credential") + + assert credential == "test-password" + + sent_requests = httpx_mock.get_requests() + assert len(sent_requests) == 2 + credential_request = sent_requests[1] + assert credential_request.method == "POST" + request_body = json.loads(credential_request.content) + assert request_body["assetName"] == "Test Credential" + assert request_body["supportsCredentialsProxyDisconnected"] is True + assert "robotKey" not in request_body + + def test_retrieve_secret( + self, + httpx_mock: HTTPXMock, + service: AssetsService, + base_url: str, + org: str, + tenant: str, + ) -> None: + """retrieve_secret returns SecretValue for Secret-type assets.""" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/odata/Assets/UiPath.Server.Configuration.OData.GetRobotAssetByNameForRobotKey", + status_code=200, + json={ + "Id": 1, + "Name": "Test Secret", + "ValueType": "Secret", + "SecretValue": "super-secret-value", + }, + ) + + secret = service.retrieve_secret(name="Test Secret") + + assert secret == "super-secret-value" + + async def test_retrieve_secret_async( + self, + httpx_mock: HTTPXMock, + service: AssetsService, + base_url: str, + org: str, + tenant: str, + ) -> None: + """retrieve_secret_async returns SecretValue for Secret-type assets.""" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/odata/Assets/UiPath.Server.Configuration.OData.GetRobotAssetByNameForRobotKey", + status_code=200, + json={ + "Id": 1, + "Name": "Test Secret", + "ValueType": "Secret", + "SecretValue": "super-secret-value", + }, + ) + + secret = await service.retrieve_secret_async(name="Test Secret") + + assert secret == "super-secret-value" + + def test_retrieve_robot_asset_exposes_secret_value( + self, + httpx_mock: HTTPXMock, + service: AssetsService, + base_url: str, + org: str, + tenant: str, + ) -> None: + """`retrieve` must expose SecretValue on UserAsset for Secret-type assets.""" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/odata/Assets/UiPath.Server.Configuration.OData.GetRobotAssetByNameForRobotKey", + status_code=200, + json={ + "Id": 1, + "Name": "Test Secret", + "ValueType": "Secret", + "SecretValue": "super-secret-value", + }, + ) + + asset = service.retrieve(name="Test Secret") + + assert isinstance(asset, UserAsset) + assert asset.value_type == "Secret" + assert asset.secret_value == "super-secret-value" + + async def test_retrieve_async_robot_asset_exposes_secret_value( + self, + httpx_mock: HTTPXMock, + service: AssetsService, + base_url: str, + org: str, + tenant: str, + ) -> None: + """`retrieve_async` must expose SecretValue on UserAsset for Secret-type assets.""" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/odata/Assets/UiPath.Server.Configuration.OData.GetRobotAssetByNameForRobotKey", + status_code=200, + json={ + "Id": 1, + "Name": "Test Secret", + "ValueType": "Secret", + "SecretValue": "super-secret-value", + }, + ) + + asset = await service.retrieve_async(name="Test Secret") + + assert isinstance(asset, UserAsset) + assert asset.value_type == "Secret" + assert asset.secret_value == "super-secret-value" + def test_update( self, httpx_mock: HTTPXMock, diff --git a/packages/uipath-platform/tests/services/test_attachments_service.py b/packages/uipath-platform/tests/services/test_attachments_service.py index dfde1a304..8e7b6aaa0 100644 --- a/packages/uipath-platform/tests/services/test_attachments_service.py +++ b/packages/uipath-platform/tests/services/test_attachments_service.py @@ -10,7 +10,7 @@ from uipath.platform import UiPathApiConfig, UiPathExecutionContext from uipath.platform.attachments import Attachment from uipath.platform.attachments.attachments import AttachmentMode -from uipath.platform.common.constants import HEADER_USER_AGENT, TEMP_ATTACHMENTS_FOLDER +from uipath.platform.constants import HEADER_USER_AGENT, TEMP_ATTACHMENTS_FOLDER from uipath.platform.orchestrator._attachments_service import AttachmentsService if TYPE_CHECKING: @@ -1193,3 +1193,17 @@ async def test_open_async_write_mode( assert upload_request is not None assert upload_request.method == "PUT" assert upload_request.url == blob_uri_response["BlobFileAccess"]["Uri"] + + +def test_attachments_service_conforms_to_attachments_protocol( + service: AttachmentsService, +) -> None: + """AttachmentsService must satisfy AttachmentsProtocol (workspace hydration). + + The static annotation makes mypy verify the upload_async overloads line up; + the isinstance check covers the runtime_checkable contract. + """ + from uipath.core.workspace import AttachmentsProtocol + + conforming: AttachmentsProtocol = service + assert isinstance(conforming, AttachmentsProtocol) diff --git a/packages/uipath-platform/tests/services/test_automation_ops_service.py b/packages/uipath-platform/tests/services/test_automation_ops_service.py new file mode 100644 index 000000000..e19a02a5a --- /dev/null +++ b/packages/uipath-platform/tests/services/test_automation_ops_service.py @@ -0,0 +1,195 @@ +"""Tests for AutomationOpsService.""" + +import json + +import httpx +import pytest +from pytest_httpx import HTTPXMock + +from uipath.platform import UiPathApiConfig, UiPathExecutionContext +from uipath.platform.automation_ops import AutomationOpsService + + +@pytest.fixture +def service( + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, +) -> AutomationOpsService: + return AutomationOpsService(config=config, execution_context=execution_context) + + +class TestAutomationOpsService: + """Test AutomationOpsService functionality.""" + + class TestGetDeployedPolicy: + """Test get_deployed_policy (sync).""" + + def test_returns_policy_dict( + self, + httpx_mock: HTTPXMock, + service: AutomationOpsService, + base_url: str, + org: str, + tenant: str, + ) -> None: + expected_policy = { + "policy-name": "AITL Policy", + "data": { + "container": {"pii-in-flight-agents": True}, + "pii-entity-table": [], + }, + } + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/agenthub_/api/policies/deployed-policy", + status_code=200, + json=expected_policy, + ) + + result = service.get_deployed_policy() + + assert result == expected_policy + + def test_returns_empty_dict_when_no_policy_deployed( + self, + httpx_mock: HTTPXMock, + service: AutomationOpsService, + base_url: str, + org: str, + tenant: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/agenthub_/api/policies/deployed-policy", + status_code=200, + content=b"", + ) + + result = service.get_deployed_policy() + + assert result == {} + + def test_uses_post_method( + self, + httpx_mock: HTTPXMock, + service: AutomationOpsService, + base_url: str, + org: str, + tenant: str, + ) -> None: + captured_request: httpx.Request | None = None + + def capture(request: httpx.Request) -> httpx.Response: + nonlocal captured_request + captured_request = request + return httpx.Response(status_code=200, json={}) + + httpx_mock.add_callback( + method="POST", + url=f"{base_url}{org}{tenant}/agenthub_/api/policies/deployed-policy", + callback=capture, + ) + + service.get_deployed_policy() + + assert captured_request is not None + assert captured_request.method == "POST" + + class TestGetDeployedPolicyAsync: + """Test get_deployed_policy_async.""" + + async def test_returns_policy_dict( + self, + httpx_mock: HTTPXMock, + service: AutomationOpsService, + base_url: str, + org: str, + tenant: str, + ) -> None: + expected_policy = { + "policy-name": "AITL Policy", + "data": { + "container": {"pii-in-flight-agents": False}, + }, + } + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/agenthub_/api/policies/deployed-policy", + status_code=200, + json=expected_policy, + ) + + result = await service.get_deployed_policy_async() + + assert result == expected_policy + + async def test_returns_empty_dict_when_no_policy_deployed( + self, + httpx_mock: HTTPXMock, + service: AutomationOpsService, + base_url: str, + org: str, + tenant: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/agenthub_/api/policies/deployed-policy", + status_code=200, + content=b"", + ) + + result = await service.get_deployed_policy_async() + + assert result == {} + + async def test_url_is_tenant_scoped( + self, + httpx_mock: HTTPXMock, + service: AutomationOpsService, + base_url: str, + org: str, + tenant: str, + ) -> None: + captured_request: httpx.Request | None = None + + def capture(request: httpx.Request) -> httpx.Response: + nonlocal captured_request + captured_request = request + return httpx.Response(status_code=200, json={}) + + httpx_mock.add_callback( + method="POST", + url=f"{base_url}{org}{tenant}/agenthub_/api/policies/deployed-policy", + callback=capture, + ) + + await service.get_deployed_policy_async() + + assert captured_request is not None + # Tenant-scoped: both org and tenant segments appear in the path + assert org.strip("/") in captured_request.url.path + assert tenant.strip("/") in captured_request.url.path + + async def test_request_has_no_body( + self, + httpx_mock: HTTPXMock, + service: AutomationOpsService, + base_url: str, + org: str, + tenant: str, + ) -> None: + captured_request: httpx.Request | None = None + + def capture(request: httpx.Request) -> httpx.Response: + nonlocal captured_request + captured_request = request + return httpx.Response(status_code=200, json={}) + + httpx_mock.add_callback( + method="POST", + url=f"{base_url}{org}{tenant}/agenthub_/api/policies/deployed-policy", + callback=capture, + ) + + await service.get_deployed_policy_async() + + assert captured_request is not None + # POST with no body — body should be empty (or an empty JSON object) + body = captured_request.content + assert body in (b"", b"null") or json.loads(body) in ({}, None) diff --git a/packages/uipath-platform/tests/services/test_azure_guardrail_validators.py b/packages/uipath-platform/tests/services/test_azure_guardrail_validators.py new file mode 100644 index 000000000..fd6a3f39e --- /dev/null +++ b/packages/uipath-platform/tests/services/test_azure_guardrail_validators.py @@ -0,0 +1,146 @@ +"""Tests for the Azure-provided guardrail validators. + +Covers HarmfulContentValidator, IntellectualPropertyValidator, and +UserPromptAttacksValidator — verifying guardrail construction, parameter +serialization, stage enforcement, and input validation. +""" + +from __future__ import annotations + +import pytest + +from uipath.platform.guardrails.decorators import ( + GuardrailExecutionStage, + HarmfulContentEntity, + HarmfulContentEntityType, + HarmfulContentValidator, + IntellectualPropertyEntityType, + IntellectualPropertyValidator, + UserPromptAttacksValidator, +) + +# --------------------------------------------------------------------------- +# HarmfulContentValidator +# --------------------------------------------------------------------------- + + +class TestHarmfulContentValidator: + """Tests for HarmfulContentValidator.""" + + def test_builds_guardrail(self): + """Verify get_built_in_guardrail returns correct structure.""" + validator = HarmfulContentValidator( + entities=[ + HarmfulContentEntity(HarmfulContentEntityType.VIOLENCE, threshold=3), + HarmfulContentEntity(HarmfulContentEntityType.HATE, threshold=4), + ] + ) + guardrail = validator.get_built_in_guardrail( + name="Test HC", + description="test", + enabled_for_evals=True, + ) + assert guardrail.validator_type == "harmful_content" + assert len(guardrail.validator_parameters) == 2 + + enum_param = guardrail.validator_parameters[0] + assert enum_param.id == "harmfulContentEntities" + assert enum_param.value == ["Violence", "Hate"] + + map_param = guardrail.validator_parameters[1] + assert map_param.id == "harmfulContentEntityThresholds" + assert map_param.value == {"Violence": 3, "Hate": 4} + + def test_empty_entities_raises(self): + """Empty entities should raise ValueError.""" + with pytest.raises(ValueError, match="non-empty"): + HarmfulContentValidator(entities=[]) + + def test_threshold_validation(self): + """Threshold outside 0-6 should raise ValueError.""" + with pytest.raises(ValueError, match="between 0 and 6"): + HarmfulContentEntity(HarmfulContentEntityType.VIOLENCE, threshold=7) + with pytest.raises(ValueError, match="between 0 and 6"): + HarmfulContentEntity(HarmfulContentEntityType.VIOLENCE, threshold=-1) + + def test_all_stages_supported(self): + """supported_stages should be empty (all stages allowed).""" + validator = HarmfulContentValidator( + entities=[HarmfulContentEntity(HarmfulContentEntityType.VIOLENCE)] + ) + assert validator.supported_stages == [] + # Should not raise for any stage + validator.validate_stage(GuardrailExecutionStage.PRE) + validator.validate_stage(GuardrailExecutionStage.POST) + + +# --------------------------------------------------------------------------- +# IntellectualPropertyValidator +# --------------------------------------------------------------------------- + + +class TestIntellectualPropertyValidator: + """Tests for IntellectualPropertyValidator.""" + + def test_builds_guardrail(self): + """Verify get_built_in_guardrail returns correct structure.""" + validator = IntellectualPropertyValidator( + entities=[ + IntellectualPropertyEntityType.TEXT, + IntellectualPropertyEntityType.CODE, + ] + ) + guardrail = validator.get_built_in_guardrail( + name="Test IP", + description=None, + enabled_for_evals=False, + ) + assert guardrail.validator_type == "intellectual_property" + assert len(guardrail.validator_parameters) == 1 + + param = guardrail.validator_parameters[0] + assert param.id == "ipEntities" + assert param.value == ["Text", "Code"] + + def test_empty_entities_raises(self): + """Empty entities should raise ValueError.""" + with pytest.raises(ValueError, match="non-empty"): + IntellectualPropertyValidator(entities=[]) + + def test_post_only(self): + """Should only support POST stage.""" + validator = IntellectualPropertyValidator( + entities=[IntellectualPropertyEntityType.TEXT] + ) + assert validator.supported_stages == [GuardrailExecutionStage.POST] + validator.validate_stage(GuardrailExecutionStage.POST) + with pytest.raises(ValueError, match="does not support stage"): + validator.validate_stage(GuardrailExecutionStage.PRE) + + +# --------------------------------------------------------------------------- +# UserPromptAttacksValidator +# --------------------------------------------------------------------------- + + +class TestUserPromptAttacksValidator: + """Tests for UserPromptAttacksValidator.""" + + def test_builds_guardrail(self): + """Verify get_built_in_guardrail returns correct structure.""" + validator = UserPromptAttacksValidator() + guardrail = validator.get_built_in_guardrail( + name="Test UPA", + description=None, + enabled_for_evals=True, + ) + assert guardrail.validator_type == "user_prompt_attacks" + assert guardrail.validator_parameters == [] + + def test_pre_only(self): + """Should only support PRE stage.""" + validator = UserPromptAttacksValidator() + assert validator.supported_stages == [GuardrailExecutionStage.PRE] + validator.validate_stage(GuardrailExecutionStage.PRE) + with pytest.raises(ValueError, match="does not support stage"): + validator.validate_stage(GuardrailExecutionStage.POST) diff --git a/packages/uipath-platform/tests/services/test_base_service.py b/packages/uipath-platform/tests/services/test_base_service.py index bfe78fd18..db39ed313 100644 --- a/packages/uipath-platform/tests/services/test_base_service.py +++ b/packages/uipath-platform/tests/services/test_base_service.py @@ -3,7 +3,7 @@ from uipath.platform import UiPathApiConfig, UiPathExecutionContext from uipath.platform.common._base_service import BaseService -from uipath.platform.common.constants import HEADER_USER_AGENT +from uipath.platform.constants import HEADER_USER_AGENT from uipath.platform.errors import EnrichedException @@ -18,6 +18,14 @@ class TestBaseService: def test_init_base_service(self, service: BaseService): assert service is not None + @pytest.mark.parametrize("anyio_backend", ["asyncio", "trio"]) + @pytest.mark.anyio + async def test_aclose_closes_owned_http_clients(self, service: BaseService): + await service.aclose() + + assert service._client.is_closed + assert service._client_async.is_closed + def test_base_service_default_headers(self, service: BaseService, secret: str): assert service.default_headers == { "Accept": "application/json", diff --git a/packages/uipath-platform/tests/services/test_buckets_service.py b/packages/uipath-platform/tests/services/test_buckets_service.py index 0fbb5f974..464d675d2 100644 --- a/packages/uipath-platform/tests/services/test_buckets_service.py +++ b/packages/uipath-platform/tests/services/test_buckets_service.py @@ -27,6 +27,18 @@ def temp_file(tmp_path): class TestBucketsService: + @pytest.mark.parametrize("anyio_backend", ["asyncio", "trio"]) + @pytest.mark.anyio + async def test_aclose_closes_all_owned_http_clients( + self, service: BucketsService + ) -> None: + await service.aclose() + + assert service._client.is_closed + assert service._client_async.is_closed + assert service.custom_client.is_closed + assert service.custom_client_async.is_closed + class TestRetrieve: def test_retrieve_by_key( self, @@ -646,6 +658,90 @@ async def test_create_async( assert bucket.id == 1 +class TestDelete: + """Tests for delete() / delete_async(). + + Regression coverage for UV-14977: delete() must build the folder header + from the folder_path/folder_key arguments (via header_folder), not solely + from the UIPATH_FOLDER_PATH / UIPATH_FOLDER_KEY env vars. + """ + + def test_delete_by_name_uses_folder_path_arg( + self, + httpx_mock: HTTPXMock, + service: BucketsService, + base_url: str, + org: str, + tenant: str, + ): + """delete(name=..., folder_path=...) sends the arg folder header on DELETE.""" + # retrieve() locates the bucket in the target folder + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/odata/Buckets?$filter=Name eq 'old-storage'&$top=1", + status_code=200, + json={"value": [{"Id": 203380, "Name": "old-storage", "Identifier": "id"}]}, + match_headers={"x-uipath-folderpath": "Playground"}, + ) + # the DELETE must carry the arg folder header (not the env-var fallback) + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/odata/Buckets(203380)", + method="DELETE", + status_code=204, + match_headers={"x-uipath-folderpath": "Playground"}, + ) + + service.delete(name="old-storage", folder_path="Playground") + + def test_delete_by_key_uses_folder_key_arg( + self, + httpx_mock: HTTPXMock, + service: BucketsService, + base_url: str, + org: str, + tenant: str, + ): + """delete(key=..., folder_key=...) sends the arg folder-key header on DELETE.""" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/odata/Buckets/UiPath.Server.Configuration.OData.GetByKey(identifier='bucket-key')", + status_code=200, + json={"value": [{"Id": 55, "Name": "kbucket", "Identifier": "bucket-key"}]}, + match_headers={"x-uipath-folderkey": "folder-123"}, + ) + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/odata/Buckets(55)", + method="DELETE", + status_code=204, + match_headers={"x-uipath-folderkey": "folder-123"}, + ) + + service.delete(key="bucket-key", folder_key="folder-123") + + @pytest.mark.asyncio + async def test_delete_async_uses_folder_path_arg( + self, + httpx_mock: HTTPXMock, + service: BucketsService, + base_url: str, + org: str, + tenant: str, + ): + """Async version honors the folder_path argument on DELETE.""" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/odata/Buckets?$filter=Name eq 'old-storage'&$top=1", + status_code=200, + json={"value": [{"Id": 99, "Name": "old-storage", "Identifier": "id"}]}, + match_headers={"x-uipath-folderpath": "Playground"}, + ) + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/odata/Buckets(99)", + method="DELETE", + status_code=204, + match_headers={"x-uipath-folderpath": "Playground"}, + ) + + await service.delete_async(name="old-storage", folder_path="Playground") + + class TestEdgeCases: """Tests for edge cases and error handling.""" diff --git a/packages/uipath-platform/tests/services/test_connections_service.py b/packages/uipath-platform/tests/services/test_connections_service.py index 75c89b517..af74984b8 100644 --- a/packages/uipath-platform/tests/services/test_connections_service.py +++ b/packages/uipath-platform/tests/services/test_connections_service.py @@ -8,7 +8,6 @@ from uipath.platform import UiPathApiConfig, UiPathExecutionContext from uipath.platform.common import jsonschema_to_pydantic -from uipath.platform.common.constants import HEADER_FOLDER_KEY, HEADER_USER_AGENT from uipath.platform.connections import ( ActivityMetadata, ActivityParameterLocationInfo, @@ -17,7 +16,11 @@ ConnectionToken, EventArguments, ) -from uipath.platform.connections._connections_service import ConnectionsService +from uipath.platform.connections._connections_service import ( + HEADER_ACTIVITY_JOB_ID, + ConnectionsService, +) +from uipath.platform.constants import HEADER_FOLDER_KEY, HEADER_USER_AGENT from uipath.platform.orchestrator._folder_service import FolderService @@ -1421,6 +1424,60 @@ def test_invoke_activity_sets_standard_headers( assert sent_request.headers["x-uipath-originator"] == "uipath-python" assert sent_request.headers["x-uipath-source"] == "uipath-python" + def test_invoke_activity_propagates_job_id_header( + self, + httpx_mock: HTTPXMock, + service: ConnectionsService, + simple_activity_metadata: ActivityMetadata, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Activity invocations carry x-uipath-job-id so GenAI calls can be stitched for licensing.""" + monkeypatch.setenv("UIPATH_JOB_KEY", "job-key-abc") + connection_id = "test-connection-123" + + httpx_mock.add_response( + method="GET", + status_code=200, + json={"id": connection_id, "name": "Test", "elementInstanceId": 1}, + ) + httpx_mock.add_response(method="POST", status_code=200, json={}) + + service.invoke_activity( + activity_metadata=simple_activity_metadata, + connection_id=connection_id, + activity_input={"body_field1": "x"}, + ) + + sent_request = httpx_mock.get_requests()[1] + assert sent_request.headers[HEADER_ACTIVITY_JOB_ID] == "job-key-abc" + + def test_invoke_activity_omits_job_id_header_when_unset( + self, + httpx_mock: HTTPXMock, + service: ConnectionsService, + simple_activity_metadata: ActivityMetadata, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """No job-id header is sent when UIPATH_JOB_KEY is not set.""" + monkeypatch.delenv("UIPATH_JOB_KEY", raising=False) + connection_id = "test-connection-123" + + httpx_mock.add_response( + method="GET", + status_code=200, + json={"id": connection_id, "name": "Test", "elementInstanceId": 1}, + ) + httpx_mock.add_response(method="POST", status_code=200, json={}) + + service.invoke_activity( + activity_metadata=simple_activity_metadata, + connection_id=connection_id, + activity_input={"body_field1": "x"}, + ) + + sent_request = httpx_mock.get_requests()[1] + assert HEADER_ACTIVITY_JOB_ID not in sent_request.headers + def test_invoke_activity_with_body_fields( self, httpx_mock: HTTPXMock, @@ -2116,3 +2173,173 @@ async def test_invoke_activity_async_uses_connection_id_from_retrieve_response( assert f"/element/instances/{original_connection_id}/" not in str( activity_request.url ) + + +def _multipart_part(body: bytes, boundary: str, name: str) -> str: + """Return the raw text of the multipart part with the given form-field name.""" + text = body.decode("utf-8", errors="replace") + for part in text.split(f"--{boundary}"): + if f'name="{name}"' in part: + return part + raise AssertionError(f"part {name!r} not found in multipart body") + + +class TestMultipartFileUpload: + """Regression tests for the multipart serializer that handles file uploads. + + Before this fix, ``_build_activity_request_spec`` always built + ``files[key] = (key, val, None)``, using the form-field name as the + multipart filename and dropping the content type. Downstream services + (e.g. Coupa's ``add_attachment`` endpoint) ended up storing every + attachment with the literal name ``attachment[file]`` and no extension. + + The serializer now branches on the value type: + + * tuple → passed through (caller controls filename + content type) + * bytes → legacy fallback, key as filename, octet-stream content type + * scalar → plain multipart form field (no filename in Content-Disposition) + """ + + def test_invoke_activity_multipart_tuple_3_preserves_filename( + self, + httpx_mock: HTTPXMock, + service: ConnectionsService, + multipart_activity_metadata: ActivityMetadata, + ) -> None: + """3-tuple input is forwarded verbatim, so the real filename + content type land on the wire.""" + connection_id = "test-connection-123" + activity_input = { + "file_param": ("invoice.pdf", b"%PDF-1.4 fake", "application/pdf"), + "description": "Test file upload", + } + + httpx_mock.add_response( + method="GET", + status_code=200, + json={"id": connection_id, "name": "Test", "elementInstanceId": 1}, + ) + httpx_mock.add_response(method="POST", status_code=200, json={"ok": True}) + + _ = service.invoke_activity( + activity_metadata=multipart_activity_metadata, + connection_id=connection_id, + activity_input=activity_input, + ) + + sent_request = httpx_mock.get_requests()[1] + boundary = sent_request.headers["content-type"].split("boundary=")[1] + part = _multipart_part(sent_request.content, boundary, "file_param") + + assert 'filename="invoice.pdf"' in part + assert "Content-Type: application/pdf" in part + assert b"%PDF-1.4 fake" in sent_request.content + + def test_invoke_activity_multipart_tuple_2_preserves_filename( + self, + httpx_mock: HTTPXMock, + service: ConnectionsService, + multipart_activity_metadata: ActivityMetadata, + ) -> None: + """2-tuple (filename, content) shorthand: filename preserved, httpx infers the content type.""" + connection_id = "test-connection-123" + activity_input = { + "file_param": ("invoice.pdf", b"%PDF-1.4 fake"), + "description": "Test file upload", + } + + httpx_mock.add_response( + method="GET", + status_code=200, + json={"id": connection_id, "name": "Test", "elementInstanceId": 1}, + ) + httpx_mock.add_response(method="POST", status_code=200, json={"ok": True}) + + _ = service.invoke_activity( + activity_metadata=multipart_activity_metadata, + connection_id=connection_id, + activity_input=activity_input, + ) + + sent_request = httpx_mock.get_requests()[1] + boundary = sent_request.headers["content-type"].split("boundary=")[1] + part = _multipart_part(sent_request.content, boundary, "file_param") + + assert 'filename="invoice.pdf"' in part + assert b"%PDF-1.4 fake" in sent_request.content + + def test_invoke_activity_multipart_bytes_backwards_compatible( + self, + httpx_mock: HTTPXMock, + service: ConnectionsService, + multipart_activity_metadata: ActivityMetadata, + ) -> None: + """Existing callers passing raw bytes keep working — filename = form-field name (legacy).""" + connection_id = "test-connection-123" + activity_input = { + "file_param": b"raw bytes", + "description": "Test", + } + + httpx_mock.add_response( + method="GET", + status_code=200, + json={"id": connection_id, "name": "Test", "elementInstanceId": 1}, + ) + httpx_mock.add_response(method="POST", status_code=200, json={"ok": True}) + + _ = service.invoke_activity( + activity_metadata=multipart_activity_metadata, + connection_id=connection_id, + activity_input=activity_input, + ) + + sent_request = httpx_mock.get_requests()[1] + boundary = sent_request.headers["content-type"].split("boundary=")[1] + part = _multipart_part(sent_request.content, boundary, "file_param") + + # Legacy fallback: form-field name used as filename, octet-stream content type. + assert 'filename="file_param"' in part + assert "Content-Type: application/octet-stream" in part + assert b"raw bytes" in sent_request.content + + def test_invoke_activity_multipart_scalar_is_plain_form_field( + self, + httpx_mock: HTTPXMock, + service: ConnectionsService, + ) -> None: + """Scalar multipart_params get sent as plain form fields (no bogus filename).""" + metadata = ActivityMetadata( + object_path="/elements/test-connector/upload", + method_name="POST", + content_type="multipart/form-data", + parameter_location_info=ActivityParameterLocationInfo( + multipart_params=["file_param", "payload"], + body_fields=[], + ), + ) + connection_id = "test-connection-123" + activity_input = { + "file_param": ("doc.pdf", b"data", "application/pdf"), + "payload": "{}", + } + + httpx_mock.add_response( + method="GET", + status_code=200, + json={"id": connection_id, "name": "Test", "elementInstanceId": 1}, + ) + httpx_mock.add_response(method="POST", status_code=200, json={"ok": True}) + + _ = service.invoke_activity( + activity_metadata=metadata, + connection_id=connection_id, + activity_input=activity_input, + ) + + sent_request = httpx_mock.get_requests()[1] + boundary = sent_request.headers["content-type"].split("boundary=")[1] + payload_part = _multipart_part(sent_request.content, boundary, "payload") + + # Scalar payload must NOT carry a filename in Content-Disposition. + assert "filename=" not in payload_part + assert "{}" in payload_part diff --git a/packages/uipath-platform/tests/services/test_context_grounding_service.py b/packages/uipath-platform/tests/services/test_context_grounding_service.py index 135ac281b..b98942769 100644 --- a/packages/uipath-platform/tests/services/test_context_grounding_service.py +++ b/packages/uipath-platform/tests/services/test_context_grounding_service.py @@ -6,7 +6,11 @@ from pytest_httpx import HTTPXMock from uipath.platform import UiPathApiConfig, UiPathExecutionContext -from uipath.platform.common.constants import HEADER_USER_AGENT +from uipath.platform.constants import ( + ENV_JOB_KEY, + HEADER_JOB_KEY, + HEADER_USER_AGENT, +) from uipath.platform.context_grounding import ( BatchTransformCreationResponse, BatchTransformOutputColumn, @@ -31,6 +35,7 @@ from uipath.platform.context_grounding._context_grounding_service import ( ContextGroundingService, ) +from uipath.platform.errors import ContextGroundingIndexNotFoundError from uipath.platform.orchestrator._buckets_service import BucketsService from uipath.platform.orchestrator._folder_service import FolderService @@ -761,6 +766,529 @@ async def test_retrieve_async_falls_back_to_across_folders_when_no_folder_contex assert len(sent_requests) == 1 assert "/ecs_/v2/indexes/allacrossfolders" in str(sent_requests[0].url) + def test_retrieve_system_indexes( + self, + httpx_mock: HTTPXMock, + service: ContextGroundingService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/allsystemindexes?$expand=dataSource", + status_code=200, + json={ + "value": [ + { + "id": "sys-index-1", + "name": "system-template-index", + "lastIngestionStatus": "Completed", + }, + { + "id": "sys-index-2", + "name": "system-other-index", + "lastIngestionStatus": "Completed", + }, + ] + }, + ) + + indexes = service._retrieve_system_indexes() + + assert isinstance(indexes, list) + assert len(indexes) == 2 + assert isinstance(indexes[0], ContextGroundingIndex) + assert indexes[0].id == "sys-index-1" + assert indexes[0].name == "system-template-index" + + sent_requests = httpx_mock.get_requests() + assert sent_requests[0].method == "GET" + assert "/ecs_/v2/indexes/allsystemindexes" in str(sent_requests[0].url) + assert "x-uipath-folderkey" not in sent_requests[0].headers + + assert HEADER_USER_AGENT in sent_requests[0].headers + assert ( + sent_requests[0].headers[HEADER_USER_AGENT] + == f"UiPath.Python.Sdk/UiPath.Python.Sdk.Activities.ContextGroundingService._retrieve_system_indexes/{version}" + ) + + def test_retrieve_system_indexes_with_name_filter( + self, + httpx_mock: HTTPXMock, + service: ContextGroundingService, + base_url: str, + org: str, + tenant: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/allsystemindexes?$expand=dataSource&$filter=Name eq 'system-template-index'", + status_code=200, + json={ + "value": [ + { + "id": "sys-index-1", + "name": "system-template-index", + "lastIngestionStatus": "Completed", + }, + ] + }, + ) + + indexes = service._retrieve_system_indexes(name="system-template-index") + + assert len(indexes) == 1 + assert indexes[0].name == "system-template-index" + + sent_requests = httpx_mock.get_requests() + assert "allsystemindexes" in str(sent_requests[0].url) + assert "x-uipath-folderkey" not in sent_requests[0].headers + + @pytest.mark.anyio + async def test_retrieve_system_indexes_async( + self, + httpx_mock: HTTPXMock, + service: ContextGroundingService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/allsystemindexes?$expand=dataSource", + status_code=200, + json={ + "value": [ + { + "id": "sys-index-1", + "name": "system-template-index", + "lastIngestionStatus": "Completed", + } + ] + }, + ) + + indexes = await service._retrieve_system_indexes_async() + + assert len(indexes) == 1 + assert indexes[0].id == "sys-index-1" + + sent_requests = httpx_mock.get_requests() + assert sent_requests[0].method == "GET" + assert "/ecs_/v2/indexes/allsystemindexes" in str(sent_requests[0].url) + assert "x-uipath-folderkey" not in sent_requests[0].headers + + assert ( + sent_requests[0].headers[HEADER_USER_AGENT] + == f"UiPath.Python.Sdk/UiPath.Python.Sdk.Activities.ContextGroundingService._retrieve_system_indexes_async/{version}" + ) + + def test_retrieve_system_indexes_escapes_single_quote_in_name( + self, + httpx_mock: HTTPXMock, + service: ContextGroundingService, + base_url: str, + org: str, + tenant: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/allsystemindexes?$expand=dataSource&$filter=Name eq 'O''Brien'", + status_code=200, + json={ + "value": [ + { + "id": "sys-1", + "name": "O'Brien", + "lastIngestionStatus": "Completed", + } + ] + }, + ) + + indexes = service._retrieve_system_indexes(name="O'Brien") + + assert len(indexes) == 1 + assert indexes[0].name == "O'Brien" + + def test_retrieve_across_folders_escapes_single_quote_in_name( + self, + httpx_mock: HTTPXMock, + service: ContextGroundingService, + base_url: str, + org: str, + tenant: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/allacrossfolders?$expand=dataSource&$filter=Name eq 'O''Brien'", + status_code=200, + json={ + "value": [ + { + "id": "idx-1", + "name": "O'Brien", + "lastIngestionStatus": "Completed", + } + ] + }, + ) + + indexes = service.retrieve_across_folders(name="O'Brien") + + assert len(indexes) == 1 + assert indexes[0].name == "O'Brien" + + def test_retrieve_system_indexes_empty( + self, + httpx_mock: HTTPXMock, + service: ContextGroundingService, + base_url: str, + org: str, + tenant: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/allsystemindexes?$expand=dataSource", + status_code=200, + json={"value": []}, + ) + + indexes = service._retrieve_system_indexes() + + assert indexes == [] + + def test_retrieve_raises_typed_not_found_when_across_folders_empty( + self, + httpx_mock: HTTPXMock, + service_no_folder: ContextGroundingService, + base_url: str, + org: str, + tenant: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/allacrossfolders?$expand=dataSource&$filter=Name eq 'missing-index'", + status_code=200, + json={"value": []}, + ) + + with pytest.raises(ContextGroundingIndexNotFoundError) as exc_info: + service_no_folder.retrieve(name="missing-index") + + assert exc_info.value.index_name == "missing-index" + + @pytest.mark.anyio + async def test_retrieve_async_raises_typed_not_found_when_across_folders_empty( + self, + httpx_mock: HTTPXMock, + service_no_folder: ContextGroundingService, + base_url: str, + org: str, + tenant: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/allacrossfolders?$expand=dataSource&$filter=Name eq 'missing-index'", + status_code=200, + json={"value": []}, + ) + + with pytest.raises(ContextGroundingIndexNotFoundError): + await service_no_folder.retrieve_async(name="missing-index") + + def test_retrieve_falls_back_to_system_indexes_when_flag_true( + self, + httpx_mock: HTTPXMock, + service_no_folder: ContextGroundingService, + base_url: str, + org: str, + tenant: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/allacrossfolders?$expand=dataSource&$filter=Name eq 'system-template-index'", + status_code=200, + json={"value": []}, + ) + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/allsystemindexes?$expand=dataSource&$filter=Name eq 'system-template-index'", + status_code=200, + json={ + "value": [ + { + "id": "sys-1", + "name": "system-template-index", + "lastIngestionStatus": "Completed", + } + ] + }, + ) + + index = service_no_folder.retrieve( + name="system-template-index", include_system_indexes=True + ) + + assert index.id == "sys-1" + assert index.name == "system-template-index" + + sent_requests = httpx_mock.get_requests() + assert len(sent_requests) == 2 + assert "/ecs_/v2/indexes/allacrossfolders" in str(sent_requests[0].url) + assert "/ecs_/v2/indexes/allsystemindexes" in str(sent_requests[1].url) + + def test_retrieve_does_not_fall_back_to_system_indexes_when_flag_false( + self, + httpx_mock: HTTPXMock, + service_no_folder: ContextGroundingService, + base_url: str, + org: str, + tenant: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/allacrossfolders?$expand=dataSource&$filter=Name eq 'missing-index'", + status_code=200, + json={"value": []}, + ) + + with pytest.raises(ContextGroundingIndexNotFoundError): + service_no_folder.retrieve(name="missing-index") + + sent_requests = httpx_mock.get_requests() + assert len(sent_requests) == 1 + assert "/ecs_/v2/indexes/allacrossfolders" in str(sent_requests[0].url) + + def test_retrieve_skips_system_indexes_when_across_folders_resolves( + self, + httpx_mock: HTTPXMock, + service_no_folder: ContextGroundingService, + base_url: str, + org: str, + tenant: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/allacrossfolders?$expand=dataSource&$filter=Name eq 'tenant-index'", + status_code=200, + json={ + "value": [ + { + "id": "tenant-1", + "name": "tenant-index", + "lastIngestionStatus": "Completed", + "folderKey": "folder-x", + } + ] + }, + ) + + index = service_no_folder.retrieve( + name="tenant-index", include_system_indexes=True + ) + + assert index.id == "tenant-1" + + sent_requests = httpx_mock.get_requests() + assert len(sent_requests) == 1 + assert "/ecs_/v2/indexes/allacrossfolders" in str(sent_requests[0].url) + + def test_retrieve_falls_back_to_system_indexes_after_folder_lookup_misses( + self, + httpx_mock: HTTPXMock, + service: ContextGroundingService, + base_url: str, + org: str, + tenant: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/api/FoldersNavigation/GetFoldersForCurrentUser?searchText=test-folder-path&skip=0&take=20", + status_code=200, + json={ + "PageItems": [ + { + "Key": "test-folder-key", + "FullyQualifiedName": "test-folder-path", + } + ] + }, + ) + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes?$filter=Name eq 'system-template-index'&$expand=dataSource", + status_code=200, + json={"value": []}, + ) + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/allsystemindexes?$expand=dataSource&$filter=Name eq 'system-template-index'", + status_code=200, + json={ + "value": [ + { + "id": "sys-1", + "name": "system-template-index", + "lastIngestionStatus": "Completed", + } + ] + }, + ) + + index = service.retrieve( + name="system-template-index", + folder_path="test-folder-path", + include_system_indexes=True, + ) + + assert index.id == "sys-1" + + @pytest.mark.anyio + async def test_retrieve_async_falls_back_to_system_indexes_when_flag_true( + self, + httpx_mock: HTTPXMock, + service_no_folder: ContextGroundingService, + base_url: str, + org: str, + tenant: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/allacrossfolders?$expand=dataSource&$filter=Name eq 'system-template-index'", + status_code=200, + json={"value": []}, + ) + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/allsystemindexes?$expand=dataSource&$filter=Name eq 'system-template-index'", + status_code=200, + json={ + "value": [ + { + "id": "sys-1", + "name": "system-template-index", + "lastIngestionStatus": "Completed", + } + ] + }, + ) + + index = await service_no_folder.retrieve_async( + name="system-template-index", include_system_indexes=True + ) + + assert index.id == "sys-1" + + def test_retrieve_with_flag_raises_when_system_indexes_also_empty( + self, + httpx_mock: HTTPXMock, + service_no_folder: ContextGroundingService, + base_url: str, + org: str, + tenant: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/allacrossfolders?$expand=dataSource&$filter=Name eq 'missing-index'", + status_code=200, + json={"value": []}, + ) + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/allsystemindexes?$expand=dataSource&$filter=Name eq 'missing-index'", + status_code=200, + json={"value": []}, + ) + + with pytest.raises(ContextGroundingIndexNotFoundError): + service_no_folder.retrieve( + name="missing-index", include_system_indexes=True + ) + + def test_unified_search_forwards_include_system_indexes( + self, + httpx_mock: HTTPXMock, + service_no_folder: ContextGroundingService, + base_url: str, + org: str, + tenant: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/allacrossfolders?$expand=dataSource&$filter=Name eq 'system-template-index'", + status_code=200, + json={"value": []}, + ) + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/allsystemindexes?$expand=dataSource&$filter=Name eq 'system-template-index'", + status_code=200, + json={ + "value": [ + { + "id": "sys-1", + "name": "system-template-index", + "lastIngestionStatus": "Completed", + } + ] + }, + ) + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v1.2/search/sys-1", + status_code=200, + json={ + "semanticResults": { + "values": [], + "metadata": {"operation_id": "op-1", "strategy": "semantic"}, + } + }, + ) + + result = service_no_folder.unified_search( + name="system-template-index", + query="hello", + include_system_indexes=True, + ) + + assert isinstance(result, UnifiedQueryResult) + + sent_requests = httpx_mock.get_requests() + assert any("allsystemindexes" in str(r.url) for r in sent_requests) + assert any("search/sys-1" in str(r.url) for r in sent_requests) + + @pytest.mark.anyio + async def test_unified_search_async_forwards_include_system_indexes( + self, + httpx_mock: HTTPXMock, + service_no_folder: ContextGroundingService, + base_url: str, + org: str, + tenant: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/allacrossfolders?$expand=dataSource&$filter=Name eq 'system-template-index'", + status_code=200, + json={"value": []}, + ) + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/allsystemindexes?$expand=dataSource&$filter=Name eq 'system-template-index'", + status_code=200, + json={ + "value": [ + { + "id": "sys-1", + "name": "system-template-index", + "lastIngestionStatus": "Completed", + } + ] + }, + ) + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v1.2/search/sys-1", + status_code=200, + json={ + "semanticResults": { + "values": [], + "metadata": {"operation_id": "op-1", "strategy": "semantic"}, + } + }, + ) + + result = await service_no_folder.unified_search_async( + name="system-template-index", + query="hello", + include_system_indexes=True, + ) + + assert isinstance(result, UnifiedQueryResult) + + sent_requests = httpx_mock.get_requests() + assert any("allsystemindexes" in str(r.url) for r in sent_requests) + assert any("search/sys-1" in str(r.url) for r in sent_requests) + def test_search_uses_index_folder_key_when_no_folder_context( self, httpx_mock: HTTPXMock, @@ -2749,6 +3277,71 @@ async def test_create_ephemeral_index_async( == f"UiPath.Python.Sdk/UiPath.Python.Sdk.Activities.ContextGroundingService.create_ephemeral_index_async/{version}" ) + def test_create_ephemeral_index_with_folder_key( + self, + httpx_mock: HTTPXMock, + service: ContextGroundingService, + base_url: str, + org: str, + tenant: str, + ) -> None: + import uuid + + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/createephemeral", + status_code=200, + json={ + "id": "ephemeral-index-id", + "name": "ephemeral-index", + "lastIngestionStatus": "Queued", + }, + ) + + attachment_ids = [str(uuid.uuid4())] + service.create_ephemeral_index( + usage="DeepRAG", + attachments=attachment_ids, + folder_key="test-folder-key", + ) + + sent_requests = httpx_mock.get_requests() + assert sent_requests is not None + assert "x-uipath-folderkey" in sent_requests[0].headers + assert sent_requests[0].headers["x-uipath-folderkey"] == "test-folder-key" + + @pytest.mark.anyio + async def test_create_ephemeral_index_async_with_folder_key( + self, + httpx_mock: HTTPXMock, + service: ContextGroundingService, + base_url: str, + org: str, + tenant: str, + ) -> None: + import uuid + + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/indexes/createephemeral", + status_code=200, + json={ + "id": "ephemeral-index-id", + "name": "ephemeral-index", + "lastIngestionStatus": "Queued", + }, + ) + + attachment_ids = [str(uuid.uuid4())] + await service.create_ephemeral_index_async( + usage="DeepRAG", + attachments=attachment_ids, + folder_key="test-folder-key", + ) + + sent_requests = httpx_mock.get_requests() + assert sent_requests is not None + assert "x-uipath-folderkey" in sent_requests[0].headers + assert sent_requests[0].headers["x-uipath-folderkey"] == "test-folder-key" + @pytest.mark.anyio async def test_download_batch_transform_result_async_creates_nested_directories( self, @@ -3111,7 +3704,7 @@ async def test_unified_search_async( response = await service.unified_search_async( name="test-index", query="test query", - search_mode=SearchMode.AUTO, + search_mode=SearchMode.SEMANTIC, ) assert isinstance(response, UnifiedQueryResult) @@ -3193,3 +3786,135 @@ def test_unified_search_with_scope( assert "filter" not in request_body assert request_body["scope"]["folder"] == "docs" assert request_body["scope"]["extension"] == ".pdf" + + +class TestJobKeyHeader: + """X-UiPath-JobKey is attached to outbound ECS calls when UIPATH_JOB_KEY is set.""" + + _INDEX = ContextGroundingIndex( + id="test-index-id", + name="test-index", + last_ingestion_status="Completed", + ) + + def test_ingest_data_carries_job_key_header( + self, + service: ContextGroundingService, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv(ENV_JOB_KEY, "job-key-ingest") + + with patch.object(service, "request") as mock_request: + mock_request.return_value = MagicMock() + service.ingest_data(self._INDEX, folder_key="test-folder-key") + + headers = mock_request.call_args[1]["headers"] + assert headers[HEADER_JOB_KEY] == "job-key-ingest" + + @pytest.mark.anyio + async def test_ingest_data_async_carries_job_key_header( + self, + service: ContextGroundingService, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv(ENV_JOB_KEY, "job-key-ingest-async") + + with patch.object(service, "request_async") as mock_request: + mock_request.return_value = MagicMock() + await service.ingest_data_async(self._INDEX, folder_key="test-folder-key") + + headers = mock_request.call_args[1]["headers"] + assert headers[HEADER_JOB_KEY] == "job-key-ingest-async" + + def test_unified_search_carries_job_key_header( + self, + service: ContextGroundingService, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv(ENV_JOB_KEY, "job-key-search") + + with patch.object(service, "request") as mock_request: + mock_response = MagicMock() + mock_response.json.return_value = { + "semanticResults": {"values": []}, + "explanation": None, + } + mock_request.return_value = mock_response + with patch.object(service, "retrieve", return_value=self._INDEX): + service.unified_search( + name="test-index", + query="test query", + folder_key="test-folder-key", + ) + + headers = mock_request.call_args[1]["headers"] + assert headers[HEADER_JOB_KEY] == "job-key-search" + + def test_start_deep_rag_carries_job_key_header( + self, + service: ContextGroundingService, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv(ENV_JOB_KEY, "job-key-deeprag") + + with patch.object(service, "request") as mock_request: + mock_response = MagicMock() + mock_response.json.return_value = { + "id": "new-deep-rag-task-id", + "lastDeepRagStatus": "Queued", + "createdDate": "2024-01-15T10:30:00Z", + } + mock_request.return_value = mock_response + service.start_deep_rag( + index_id="test-index-id", + name="my-deep-rag-task", + prompt="Summarize", + glob_pattern="*.pdf", + citation_mode=CitationMode.INLINE, + folder_key="test-folder-key", + ) + + headers = mock_request.call_args[1]["headers"] + assert headers[HEADER_JOB_KEY] == "job-key-deeprag" + + def test_start_batch_transform_carries_job_key_header( + self, + service: ContextGroundingService, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv(ENV_JOB_KEY, "job-key-batch") + + with patch.object(service, "request") as mock_request: + mock_response = MagicMock() + mock_response.json.return_value = { + "id": "new-batch-id", + "lastBatchRagStatus": "Queued", + } + mock_request.return_value = mock_response + service.start_batch_transform( + index_id="test-index-id", + name="my-batch-task", + prompt="Extract", + output_columns=[ + BatchTransformOutputColumn(name="col1", description="d") + ], + enable_web_search_grounding=False, + folder_key="test-folder-key", + ) + + headers = mock_request.call_args[1]["headers"] + assert headers[HEADER_JOB_KEY] == "job-key-batch" + + def test_ingest_data_omits_job_key_header_when_env_unset( + self, + service: ContextGroundingService, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.delenv(ENV_JOB_KEY, raising=False) + + with patch.object(service, "request") as mock_request: + mock_request.return_value = MagicMock() + service.ingest_data(self._INDEX, folder_key="test-folder-key") + + headers = mock_request.call_args[1]["headers"] + assert HEADER_JOB_KEY not in headers diff --git a/packages/uipath-platform/tests/services/test_conversations_service.py b/packages/uipath-platform/tests/services/test_conversations_service.py index 31aa4a653..37e08bdfa 100644 --- a/packages/uipath-platform/tests/services/test_conversations_service.py +++ b/packages/uipath-platform/tests/services/test_conversations_service.py @@ -38,7 +38,6 @@ async def test_retrieve_message( "role": "assistant", "contentParts": [], "toolCalls": [], - "interrupts": [], "createdAt": "2024-01-01T00:00:00Z", "updatedAt": "2024-01-01T00:00:00Z", }, @@ -95,7 +94,6 @@ async def test_retrieve_message_with_content_parts( } ], "toolCalls": [], - "interrupts": [], "createdAt": "2024-01-01T00:00:00Z", "updatedAt": "2024-01-01T00:00:00Z", }, @@ -145,7 +143,6 @@ async def test_retrieve_message_with_tool_calls( "updatedAt": "2024-01-01T00:00:00Z", } ], - "interrupts": [], "createdAt": "2024-01-01T00:00:00Z", "updatedAt": "2024-01-01T00:00:00Z", }, diff --git a/packages/uipath-platform/tests/services/test_entities_service.py b/packages/uipath-platform/tests/services/test_entities_service.py index 8a9abafef..9a63bbc46 100644 --- a/packages/uipath-platform/tests/services/test_entities_service.py +++ b/packages/uipath-platform/tests/services/test_entities_service.py @@ -1,3 +1,4 @@ +import json import re import uuid from dataclasses import make_dataclass @@ -8,8 +9,14 @@ from pytest_httpx import HTTPXMock from uipath.platform import UiPathApiConfig, UiPathExecutionContext -from uipath.platform.entities import Entity, EntityRouting, QueryRoutingOverrideContext +from uipath.platform.common._bindings import ( + EntityResourceOverwrite, + _resource_overwrites, +) +from uipath.platform.entities import ChoiceSetValue, DataFabricEntityItem, Entity from uipath.platform.entities._entities_service import EntitiesService +from uipath.platform.entities._entity_data_service import EntityDataService +from uipath.platform.errors import EnrichedException @pytest.fixture @@ -48,6 +55,33 @@ def record_schema_optional(request): class TestEntitiesService: + @pytest.mark.parametrize("anyio_backend", ["asyncio", "trio"]) + @pytest.mark.anyio + async def test_aclose_closes_owned_services(self, service: EntitiesService): + await service.aclose() + + assert service._client.is_closed + assert service._client_async.is_closed + assert service._schema._client.is_closed + assert service._schema._client_async.is_closed + assert service._data._client.is_closed + assert service._data._client_async.is_closed + assert service._ontology._client.is_closed + assert service._ontology._client_async.is_closed + + def test_query_entity_records_has_datafabric_error_mapping(self) -> None: + assert ( + EntitiesService.query_entity_records.__uipath_datafabric_method__ # type: ignore[attr-defined] + == "query_entity_records" + ) + assert ( + EntitiesService.query_entity_records.__uipath_datafabric_error_codes__ # type: ignore[attr-defined] + == EntitiesService.query_entity_records_async.__uipath_datafabric_error_codes__ # type: ignore[attr-defined] + ) + assert "SQL_PARSING" in ( + EntitiesService.query_entity_records.__uipath_datafabric_error_codes__ # type: ignore[attr-defined] + ) + def test_retrieve( self, httpx_mock: HTTPXMock, @@ -263,12 +297,52 @@ def test_retrieve_records_with_optional_fields( limit=1, ) + def test_retrieve_records_without_start_and_limit( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + entity_key = uuid.uuid4() + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{str(entity_key)}/read", + status_code=200, + json={ + "totalCount": 1, + "value": [ + {"Id": "12345", "name": "record_name", "integer_field": 10}, + ], + }, + ) + + records = service.list_records(entity_key=str(entity_key)) + + sent_request = httpx_mock.get_request() + if sent_request is None: + raise Exception("No request was sent") + + # Verify no start or limit query params are sent + assert "start" not in str(sent_request.url.params) + assert "limit" not in str(sent_request.url.params) + + assert isinstance(records, list) + assert len(records) == 1 + assert records[0].id == "12345" + @pytest.mark.parametrize( "sql_query", [ "SELECT id FROM Customers WHERE id = 1", "SELECT id, name FROM Customers LIMIT 10", - "SELECT * FROM Customers WHERE status = 'Active'", + "SELECT COUNT(id) FROM Customers", + "SELECT SUM(amount) FROM Orders", + "SELECT AVG(price) FROM Products", + "SELECT MIN(created), MAX(created) FROM Events", + "SELECT COUNT(id) AS total, SUM(amount) AS amt FROM Orders", + "SELECT COUNT(id), name FROM Customers LIMIT 10", "SELECT id, name, email, phone FROM Customers LIMIT 5", "SELECT DISTINCT id FROM Customers WHERE id > 100", "SELECT id FROM Customers WHERE name = 'foo;bar'", @@ -280,7 +354,7 @@ def test_retrieve_records_with_optional_fields( def test_validate_sql_query_allows_supported_select_queries( self, sql_query: str, service: EntitiesService ) -> None: - service._validate_sql_query(sql_query) + service._data._validate_sql_query(sql_query) @pytest.mark.parametrize( "sql_query,error_message", @@ -316,9 +390,49 @@ def test_validate_sql_query_allows_supported_select_queries( "SELECT id FROM Customers", "Queries without WHERE must include a LIMIT clause.", ), + ( + "SELECT UPPER(name) FROM Customers", + "Queries without WHERE must include a LIMIT clause.", + ), + ( + "SELECT COALESCE(name, 'N/A') FROM Customers", + "Queries without WHERE must include a LIMIT clause.", + ), + ( + "SELECT 1 LIMIT 1", + "Queries must include a FROM clause.", + ), + ( + "SELECT COUNT(*) FROM Customers", + "COUNT(*) is not supported. Use COUNT(column_name) instead.", + ), + ( + "SELECT COUNT(*), name FROM Customers LIMIT 10", + "COUNT(*) is not supported. Use COUNT(column_name) instead.", + ), + ( + "SELECT COUNT(*) AS total FROM Customers", + "COUNT(*) is not supported. Use COUNT(column_name) instead.", + ), ( "SELECT * FROM Customers LIMIT 10", - "SELECT * without filtering is not allowed.", + "SELECT * is not allowed. Specify column names instead.", + ), + ( + "SELECT Customers.* FROM Customers LIMIT 10", + "SELECT * is not allowed. Specify column names instead.", + ), + ( + "SELECT t.* FROM Customers t LIMIT 10", + "SELECT * is not allowed. Specify column names instead.", + ), + ( + "SELECT * FROM Customers WHERE status = 'Active'", + "SELECT * is not allowed. Specify column names instead.", + ), + ( + "SELECT Customers.* FROM Customers WHERE status = 'Active'", + "SELECT * is not allowed. Specify column names instead.", ), ( "SELECT id, name, email, phone, address FROM Customers LIMIT 10", @@ -330,20 +444,20 @@ def test_validate_sql_query_rejects_disallowed_queries( self, sql_query: str, error_message: str, service: EntitiesService ) -> None: with pytest.raises(ValueError, match=re.escape(error_message)): - service._validate_sql_query(sql_query) + service._data._validate_sql_query(sql_query) def test_query_entity_records_rejects_invalid_sql_before_network_call( self, service: EntitiesService, ) -> None: - service.request = MagicMock() # type: ignore[method-assign] + service._data.request = MagicMock() # type: ignore[method-assign] with pytest.raises( ValueError, match=re.escape("Only SELECT statements are allowed.") ): service.query_entity_records("UPDATE Customers SET name = 'X'") - service.request.assert_not_called() + service._data.request.assert_not_called() def test_query_entity_records_calls_request_for_valid_sql( self, @@ -352,26 +466,65 @@ def test_query_entity_records_calls_request_for_valid_sql( response = MagicMock() response.json.return_value = {"results": [{"id": 1}, {"id": 2}]} - service.request = MagicMock(return_value=response) # type: ignore[method-assign] + service._data.request = MagicMock(return_value=response) # type: ignore[method-assign] result = service.query_entity_records("SELECT id FROM Customers WHERE id > 0") assert result == [{"id": 1}, {"id": 2}] - service.request.assert_called_once() + service._data.request.assert_called_once() + + def test_query_entity_records_sets_relationships_as_scalar_option_when_true( + self, + service: EntitiesService, + ) -> None: + response = MagicMock() + response.json.return_value = {"results": []} + service._data.request = MagicMock(return_value=response) # type: ignore[method-assign] + + service.query_entity_records( + "SELECT id FROM Customers WHERE id > 0", relationships_as_scalar=True + ) + + call_kwargs = service._data.request.call_args + body = call_kwargs.kwargs.get("json") or call_kwargs[1].get("json") + assert body["queryOptions"] == {"relationshipsAsScalar": True} + + def test_query_entity_records_omits_query_options_by_default( + self, + service: EntitiesService, + ) -> None: + response = MagicMock() + response.json.return_value = {"results": []} + service._data.request = MagicMock(return_value=response) # type: ignore[method-assign] + + service.query_entity_records("SELECT id FROM Customers WHERE id > 0") + + call_kwargs = service._data.request.call_args + body = call_kwargs.kwargs.get("json") or call_kwargs[1].get("json") + assert "queryOptions" not in body + + def test_query_entity_records_flag_is_keyword_only( + self, + service: EntitiesService, + ) -> None: + # A second positional arg (as an old ``source`` call would pass) must be + # rejected rather than silently coerced into ``relationships_as_scalar``. + with pytest.raises(TypeError): + service.query_entity_records("SELECT id FROM Customers LIMIT 10", True) @pytest.mark.anyio async def test_query_entity_records_async_rejects_invalid_sql_before_network_call( self, service: EntitiesService, ) -> None: - service.request_async = AsyncMock() # type: ignore[method-assign] + service._data.request_async = AsyncMock() # type: ignore[method-assign] with pytest.raises(ValueError, match=re.escape("Subqueries are not allowed.")): await service.query_entity_records_async( "SELECT id FROM Customers WHERE id IN (SELECT id FROM Orders)" ) - service.request_async.assert_not_called() + service._data.request_async.assert_not_called() @pytest.mark.anyio async def test_query_entity_records_async_calls_request_for_valid_sql( @@ -381,77 +534,86 @@ async def test_query_entity_records_async_calls_request_for_valid_sql( response = MagicMock() response.json.return_value = {"results": [{"id": "c1"}]} - service.request_async = AsyncMock(return_value=response) # type: ignore[method-assign] + service._data.request_async = AsyncMock(return_value=response) # type: ignore[method-assign] result = await service.query_entity_records_async( "SELECT id FROM Customers WHERE id = 'c1'" ) assert result == [{"id": "c1"}] - service.request_async.assert_called_once() + service._data.request_async.assert_called_once() - def test_query_entity_records_with_routing_context( + @pytest.mark.anyio + async def test_query_entity_records_async_sets_relationships_as_scalar_option( self, service: EntitiesService, ) -> None: response = MagicMock() - response.json.return_value = {"results": [{"id": 1}]} - service.request = MagicMock(return_value=response) # type: ignore[method-assign] - - routing = QueryRoutingOverrideContext( - entity_routings=[ - EntityRouting(entity_name="Customers", folder_id="folder-1"), - EntityRouting( - entity_name="Orders", - folder_id="folder-2", - override_entity_name="OrdersV2", - ), - ] + response.json.return_value = {"results": []} + service._data.request_async = AsyncMock(return_value=response) # type: ignore[method-assign] + + await service.query_entity_records_async( + "SELECT id FROM Customers WHERE id > 0", relationships_as_scalar=True ) - result = service.query_entity_records( - "SELECT id FROM Customers LIMIT 10", routing_context=routing + call_kwargs = service._data.request_async.call_args + body = call_kwargs.kwargs.get("json") or call_kwargs[1].get("json") + assert body["queryOptions"] == {"relationshipsAsScalar": True} + + def test_query_entity_records_builds_routing_context_from_folders_map( + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + ) -> None: + service = EntitiesService( + config=config, + execution_context=execution_context, + folders_map={"Customers": "solution_folder", "Orders": "folder-2"}, ) + response = MagicMock() + response.json.return_value = {"results": [{"id": 1}]} + service._data.request = MagicMock(return_value=response) # type: ignore[method-assign] + + result = service.query_entity_records("SELECT id FROM Customers LIMIT 10") assert result == [{"id": 1}] - call_kwargs = service.request.call_args + call_kwargs = service._data.request.call_args body = call_kwargs.kwargs.get("json") or call_kwargs[1].get("json") assert body["query"] == "SELECT id FROM Customers LIMIT 10" assert body["routingContext"] == { "entityRoutings": [ - {"entityName": "Customers", "folderId": "folder-1"}, - { - "entityName": "Orders", - "folderId": "folder-2", - "overrideEntityName": "OrdersV2", - }, + {"entityName": "Customers", "folderId": "solution_folder"}, + {"entityName": "Orders", "folderId": "folder-2"}, ] } @pytest.mark.anyio - async def test_query_entity_records_async_with_routing_context( + async def test_query_entity_records_async_builds_routing_context_from_folders_map( self, - service: EntitiesService, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, ) -> None: + service = EntitiesService( + config=config, + execution_context=execution_context, + folders_map={"Customers": "solution_folder"}, + ) response = MagicMock() response.json.return_value = {"results": [{"id": "c1"}]} - service.request_async = AsyncMock(return_value=response) # type: ignore[method-assign] - - routing = QueryRoutingOverrideContext( - entity_routings=[ - EntityRouting(entity_name="Customers", folder_id="folder-1"), - ] - ) + service._data.request_async = AsyncMock(return_value=response) # type: ignore[method-assign] result = await service.query_entity_records_async( - "SELECT id FROM Customers WHERE id = 'c1'", - routing_context=routing, + "SELECT id FROM Customers WHERE id = 'c1'" ) assert result == [{"id": "c1"}] - call_kwargs = service.request_async.call_args + call_kwargs = service._data.request_async.call_args body = call_kwargs.kwargs.get("json") or call_kwargs[1].get("json") - assert "routingContext" in body + assert body["routingContext"] == { + "entityRoutings": [ + {"entityName": "Customers", "folderId": "solution_folder"}, + ] + } def test_query_entity_records_without_routing_context_omits_key( self, @@ -459,10 +621,2203 @@ def test_query_entity_records_without_routing_context_omits_key( ) -> None: response = MagicMock() response.json.return_value = {"results": []} - service.request = MagicMock(return_value=response) # type: ignore[method-assign] + service._data.request = MagicMock(return_value=response) # type: ignore[method-assign] service.query_entity_records("SELECT id FROM Customers WHERE id > 0") - call_kwargs = service.request.call_args + call_kwargs = service._data.request.call_args body = call_kwargs.kwargs.get("json") or call_kwargs[1].get("json") assert "routingContext" not in body + + def test_query_entity_records_picks_up_entity_overwrites_from_context( + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + ) -> None: + from uipath.platform.common._bindings import ( + EntityResourceOverwrite, + _resource_overwrites, + ) + + service = EntitiesService( + config=config, + execution_context=execution_context, + ) + response = MagicMock() + response.json.return_value = {"results": [{"id": 1}]} + service._data.request = MagicMock(return_value=response) # type: ignore[method-assign] + + overwrite = EntityResourceOverwrite( + resource_type="entity", + name="Overwritten Customers", + folder_id="overwritten-folder-id", + ) + token = _resource_overwrites.set({"entity.Customers": overwrite}) + try: + service.query_entity_records("SELECT id FROM Customers LIMIT 10") + finally: + _resource_overwrites.reset(token) + + call_kwargs = service._data.request.call_args + body = call_kwargs.kwargs.get("json") or call_kwargs[1].get("json") + assert body["routingContext"] == { + "entityRoutings": [ + { + "entityName": "Customers", + "folderId": "overwritten-folder-id", + "overrideEntityName": "Overwritten Customers", + }, + ] + } + + def test_query_entity_records_merges_folders_map_with_entity_name_overrides( + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + ) -> None: + service = EntitiesService( + config=config, + execution_context=execution_context, + folders_map={ + "Customers": "overwritten-folder-id", + "Orders": "orders-folder", + }, + entity_name_overrides={"Customers": "Overwritten Customers"}, + ) + response = MagicMock() + response.json.return_value = {"results": []} + service._data.request = MagicMock(return_value=response) # type: ignore[method-assign] + + service.query_entity_records("SELECT id FROM Customers LIMIT 10") + + call_kwargs = service._data.request.call_args + body = call_kwargs.kwargs.get("json") or call_kwargs[1].get("json") + routings = body["routingContext"]["entityRoutings"] + assert { + "entityName": "Customers", + "folderId": "overwritten-folder-id", + "overrideEntityName": "Overwritten Customers", + } in routings + assert {"entityName": "Orders", "folderId": "orders-folder"} in routings + # Exactly two routings — no duplicates + assert len(routings) == 2 + + def test_resolve_entity_set_uses_effective_sql_name_in_routing_context( + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + ) -> None: + service = EntitiesService( + config=config, + execution_context=execution_context, + ) + service.retrieve_by_name = MagicMock( # type: ignore[method-assign] + return_value=MagicMock(spec=Entity) + ) + + overwrite = EntityResourceOverwrite( + resource_type="entity", + name="Overwritten Customers", + folder_id="known-folder-key", + ) + token = _resource_overwrites.set({"entity.entity-1": overwrite}) + try: + resolution = service.resolve_entity_set( + [ + DataFabricEntityItem( + id="entity-1", + name="Customers", + folder_key="original-folder-key", + ) + ] + ) + finally: + _resource_overwrites.reset(token) + + assert resolution.entities_service._routing_strategy.routing_context is not None + assert resolution.entities_service._routing_strategy.routing_context.model_dump( + by_alias=True, exclude_none=True + ) == { + "entityRoutings": [ + { + "entityName": "Customers", + "folderId": "known-folder-key", + "overrideEntityName": "Overwritten Customers", + } + ] + } + service.retrieve_by_name.assert_called_once_with( + "Overwritten Customers", + "known-folder-key", + ) + + @pytest.mark.asyncio + async def test_resolve_entity_set_async_resolves_folder_paths_before_fetch( + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + ) -> None: + folders_service = MagicMock() + folders_service.retrieve_key_async = AsyncMock( + return_value="resolved-folder-id" + ) + service = EntitiesService( + config=config, + execution_context=execution_context, + folders_service=folders_service, + ) + service.retrieve_by_name_async = AsyncMock( # type: ignore[method-assign] + return_value=MagicMock(spec=Entity) + ) + + overwrite = EntityResourceOverwrite( + resource_type="entity", + name="Overwritten Customers", + folder_path="Shared/Finance", + ) + token = _resource_overwrites.set({"entity.entity-1": overwrite}) + try: + resolution = await service.resolve_entity_set_async( + [ + DataFabricEntityItem( + id="entity-1", + name="Customers", + folder_key="original-folder-key", + ) + ] + ) + finally: + _resource_overwrites.reset(token) + + folders_service.retrieve_key_async.assert_awaited_once_with( + folder_path="Shared/Finance" + ) + assert resolution.entities_service._routing_strategy.routing_context is not None + assert resolution.entities_service._routing_strategy.routing_context.model_dump( + by_alias=True, exclude_none=True + ) == { + "entityRoutings": [ + { + "entityName": "Customers", + "folderId": "resolved-folder-id", + "overrideEntityName": "Overwritten Customers", + } + ] + } + service.retrieve_by_name_async.assert_awaited_once_with( + "Overwritten Customers", + "resolved-folder-id", + ) + + def test_query_entity_records_context_overwrite_same_name_no_override_field( + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + ) -> None: + from uipath.platform.common._bindings import ( + EntityResourceOverwrite, + _resource_overwrites, + ) + + service = EntitiesService( + config=config, + execution_context=execution_context, + ) + response = MagicMock() + response.json.return_value = {"results": []} + service._data.request = MagicMock(return_value=response) # type: ignore[method-assign] + + overwrite = EntityResourceOverwrite( + resource_type="entity", + name="Customers", + folder_id="different-folder-id", + ) + token = _resource_overwrites.set({"entity.Customers": overwrite}) + try: + service.query_entity_records("SELECT id FROM Customers LIMIT 10") + finally: + _resource_overwrites.reset(token) + + call_kwargs = service._data.request.call_args + body = call_kwargs.kwargs.get("json") or call_kwargs[1].get("json") + assert body["routingContext"] == { + "entityRoutings": [ + { + "entityName": "Customers", + "folderId": "different-folder-id", + }, + ] + } + + def test_query_entity_records_resolves_overwrite_folder_path_to_folder_key( + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + ) -> None: + from uipath.platform.common._bindings import ( + EntityResourceOverwrite, + _resource_overwrites, + ) + + folders_service = MagicMock() + folders_service.retrieve_key.return_value = "resolved-folder-id" + + service = EntitiesService( + config=config, + execution_context=execution_context, + folders_service=folders_service, + ) + response = MagicMock() + response.json.return_value = {"results": []} + service._data.request = MagicMock(return_value=response) # type: ignore[method-assign] + + overwrite = EntityResourceOverwrite( + resource_type="entity", + name="Overwritten Customers", + folder_path="Shared/Finance", + ) + token = _resource_overwrites.set({"entity.Customers": overwrite}) + try: + service.query_entity_records("SELECT id FROM Customers LIMIT 10") + finally: + _resource_overwrites.reset(token) + + call_kwargs = service._data.request.call_args + body = call_kwargs.kwargs.get("json") or call_kwargs[1].get("json") + assert body["routingContext"] == { + "entityRoutings": [ + { + "entityName": "Customers", + "folderId": "resolved-folder-id", + "overrideEntityName": "Overwritten Customers", + }, + ] + } + + def test_query_entity_records_uses_folder_id_directly_without_resolution( + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + ) -> None: + from uipath.platform.common._bindings import ( + EntityResourceOverwrite, + _resource_overwrites, + ) + + folders_service = MagicMock() + folders_service.retrieve_key.return_value = None + + service = EntitiesService( + config=config, + execution_context=execution_context, + folders_service=folders_service, + ) + response = MagicMock() + response.json.return_value = {"results": []} + service._data.request = MagicMock(return_value=response) # type: ignore[method-assign] + + overwrite = EntityResourceOverwrite( + resource_type="entity", + name="Overwritten Customers", + folder_id="known-folder-key", + ) + token = _resource_overwrites.set({"entity.Customers": overwrite}) + try: + service.query_entity_records("SELECT id FROM Customers LIMIT 10") + finally: + _resource_overwrites.reset(token) + + # folder_id is a key — should NOT be sent through FolderService + folders_service.retrieve_key.assert_not_called() + + call_kwargs = service._data.request.call_args + body = call_kwargs.kwargs.get("json") or call_kwargs[1].get("json") + assert body["routingContext"] == { + "entityRoutings": [ + { + "entityName": "Customers", + "folderId": "known-folder-key", + "overrideEntityName": "Overwritten Customers", + }, + ] + } + + def test_list_choicesets( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/Entity/choiceset", + status_code=200, + json=[ + { + "name": "Status", + "displayName": "Status", + "entityType": "ChoiceSet", + "description": "Status choices", + "isRbacEnabled": False, + "id": "cs-001", + }, + { + "name": "Priority", + "displayName": "Priority", + "entityType": "ChoiceSet", + "description": "Priority levels", + "isRbacEnabled": False, + "id": "cs-002", + }, + ], + ) + + choicesets = service.list_choicesets() + + assert isinstance(choicesets, list) + assert len(choicesets) == 2 + assert choicesets[0].name == "Status" + assert choicesets[0].entity_type == "ChoiceSet" + assert choicesets[0].id == "cs-001" + assert choicesets[1].name == "Priority" + + sent_request = httpx_mock.get_request() + assert sent_request is not None + assert sent_request.method == "GET" + assert str(sent_request.url).endswith("/datafabric_/api/Entity/choiceset") + + @pytest.mark.anyio + async def test_list_choicesets_async( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/Entity/choiceset", + status_code=200, + json=[ + { + "name": "Role", + "displayName": "Role", + "entityType": "ChoiceSet", + "isRbacEnabled": False, + "id": "cs-003", + }, + ], + ) + + choicesets = await service.list_choicesets_async() + + assert len(choicesets) == 1 + assert choicesets[0].name == "Role" + assert choicesets[0].id == "cs-003" + + def test_get_choiceset_values( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + choiceset_id = "cs-001" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{choiceset_id}/query_expansion", + status_code=200, + json={ + "totalRecordCount": 3, + "jsonValue": json.dumps( + [ + { + "Id": "v1", + "Name": "Active", + "DisplayName": "Active", + "NumberId": 0, + "CreateTime": "2026-01-01T00:00:00Z", + "UpdateTime": "2026-01-01T00:00:00Z", + }, + { + "Id": "v2", + "Name": "Inactive", + "DisplayName": "Inactive", + "NumberId": 1, + "CreateTime": "2026-01-01T00:00:00Z", + "UpdateTime": "2026-01-01T00:00:00Z", + }, + { + "Id": "v3", + "Name": "Pending", + "DisplayName": "Pending", + "NumberId": 2, + }, + ] + ), + }, + ) + + values = service.get_choiceset_values(choiceset_id) + + assert isinstance(values, list) + assert len(values) == 3 + assert isinstance(values[0], ChoiceSetValue) + assert values[0].id == "v1" + assert values[0].name == "Active" + assert values[0].display_name == "Active" + assert values[0].number_id == 0 + assert values[1].number_id == 1 + assert values[2].name == "Pending" + assert values[2].created_by is None + + sent_request = httpx_mock.get_request() + assert sent_request is not None + assert sent_request.method == "POST" + + def test_get_choiceset_values_with_pagination( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + choiceset_id = "cs-001" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{choiceset_id}/query_expansion?start=0&limit=2", + status_code=200, + json={ + "totalRecordCount": 5, + "jsonValue": json.dumps( + [ + { + "Id": "v1", + "Name": "Active", + "DisplayName": "Active", + "NumberId": 0, + }, + { + "Id": "v2", + "Name": "Inactive", + "DisplayName": "Inactive", + "NumberId": 1, + }, + ] + ), + }, + ) + + values = service.get_choiceset_values(choiceset_id, start=0, limit=2) + + assert len(values) == 2 + assert values[0].name == "Active" + assert values[1].name == "Inactive" + + sent_request = httpx_mock.get_request() + assert sent_request is not None + assert "start=0" in str(sent_request.url) + assert "limit=2" in str(sent_request.url) + + @pytest.mark.anyio + async def test_get_choiceset_values_async( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + choiceset_id = "cs-002" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{choiceset_id}/query_expansion", + status_code=200, + json={ + "totalRecordCount": 1, + "jsonValue": json.dumps( + [ + { + "Id": "v1", + "Name": "ReadOnly", + "DisplayName": "Read Only", + "NumberId": 0, + }, + ] + ), + }, + ) + + values = await service.get_choiceset_values_async(choiceset_id) + + assert len(values) == 1 + assert values[0].display_name == "Read Only" + assert values[0].number_id == 0 + + def test_get_choiceset_values_empty( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + choiceset_id = "cs-empty" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{choiceset_id}/query_expansion", + status_code=200, + json={ + "totalRecordCount": 0, + "jsonValue": "[]", + }, + ) + + values = service.get_choiceset_values(choiceset_id) + + assert values == [] + + +class TestEntitiesServiceNewMethods: + """Single-record, structured-query, attachment, schema and bulk-import tests.""" + + def test_insert_record_fires_post_with_expansion_level( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + from uipath.platform.entities import EntityRecord + + entity_key = uuid.uuid4() + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/insert?expansionLevel=2", + status_code=200, + json={"Id": "rec-1", "name": "alice"}, + ) + + record = service.insert_record( + entity_key=str(entity_key), + data={"name": "alice"}, + expansion_level=2, + ) + + assert isinstance(record, EntityRecord) + assert record.id == "rec-1" + + sent = httpx_mock.get_request() + assert sent is not None + assert sent.method == "POST" + assert json.loads(sent.content) == {"name": "alice"} + + async def test_insert_record_async( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + entity_key = uuid.uuid4() + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/insert", + status_code=200, + json={"Id": "rec-1"}, + ) + + record = await service.insert_record_async( + entity_key=str(entity_key), data={"name": "bob"} + ) + assert record.id == "rec-1" + + def test_get_record( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + entity_key = uuid.uuid4() + record_id = "12345" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/read/{record_id}?expansionLevel=1", + status_code=200, + json={"Id": record_id, "name": "found"}, + ) + + record = service.get_record( + entity_key=str(entity_key), record_id=record_id, expansion_level=1 + ) + + assert record.id == record_id + + def test_update_record_accepts_dict( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + entity_key = uuid.uuid4() + record_id = "rec-9" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/update/{record_id}", + status_code=200, + json={"Id": record_id, "name": "updated"}, + ) + + record = service.update_record( + entity_key=str(entity_key), + record_id=record_id, + data={"name": "updated"}, + ) + + assert record.id == record_id + sent = httpx_mock.get_request() + assert sent is not None + assert json.loads(sent.content) == {"name": "updated"} + + def test_delete_record_uses_http_delete( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + entity_key = uuid.uuid4() + record_id = "rec-9" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/delete/{record_id}", + method="DELETE", + status_code=200, + ) + + service.delete_record(entity_key=str(entity_key), record_id=record_id) + + sent = httpx_mock.get_request() + assert sent is not None + assert sent.method == "DELETE" + + def test_query_v1_with_filter_and_pagination( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + from uipath.platform.entities import ( + EntityQueryFilter, + EntityQueryFilterGroup, + EntityQuerySortOption, + LogicalOperator, + QueryFilterOperator, + ) + + entity_key = uuid.uuid4() + httpx_mock.add_response( + url=re.compile( + rf"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/query.*" + ), + status_code=200, + json={ + "value": [{"Id": "1", "name": "alice"}, {"Id": "2", "name": "bob"}], + "totalRecordCount": 5, + }, + ) + + result = service.retrieve_records( + entity_key=str(entity_key), + filter_group=EntityQueryFilterGroup( + logical_operator=LogicalOperator.And, + query_filters=[ + EntityQueryFilter( + field_name="status", + operator=QueryFilterOperator.Equals, + value="active", + ) + ], + ), + sort_options=[EntityQuerySortOption(field_name="name", is_descending=True)], + selected_fields=["Id", "name"], + start=0, + limit=2, + expansion_level=1, + ) + + assert result.total_count == 5 + assert len(result.items) == 2 + assert result.has_next_page is True + # Backend doesn't return next_cursor on this endpoint — caller paginates + # by passing the next ``start`` themselves. + assert result.next_cursor is None + + sent = httpx_mock.get_request() + assert sent is not None + assert "/query" in str(sent.url) and "/v2/" not in str(sent.url) + # expansionLevel is a URL query param, not body + assert sent.url.params.get("expansionLevel") == "1" + body = json.loads(sent.content) + assert body["filterGroup"]["logicalOperator"] == 0 # And + assert body["filterGroup"]["queryFilters"][0]["fieldName"] == "status" + assert body["sortOptions"][0]["fieldName"] == "name" + assert body["selectedFields"] == ["Id", "name"] + # start/limit go in BODY, not as $top/$skip query params + assert body["start"] == 0 + assert body["limit"] == 2 + + def test_query_aggregate_response_handles_id_less_rows( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + """Aggregate / GROUP BY rows lack ``Id`` — parsed as :class:`AggregateRow`.""" + from uipath.platform.entities import ( + AggregateRow, + EntityAggregate, + EntityAggregateFunction, + ) + + entity_key = uuid.uuid4() + httpx_mock.add_response( + url=re.compile( + rf"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/query.*" + ), + status_code=200, + json={ + "value": [ + {"status": "active", "total": 12}, + {"status": "inactive", "total": 7}, + ], + "totalRecordCount": 2, + }, + ) + + result = service.retrieve_records( + entity_key=str(entity_key), + selected_fields=["status"], + group_by=["status"], + aggregates=[ + EntityAggregate( + function=EntityAggregateFunction.Count, + field="Id", + alias="total", + ) + ], + ) + + assert result.total_count == 2 + assert len(result.items) == 2 + # Aggregate rows lack ``Id`` and are exposed as :class:`AggregateRow`. + for row in result.items: + assert isinstance(row, AggregateRow) + assert result.items[0].status == "active" + assert result.items[0].total == 12 + sent = httpx_mock.get_request() + assert sent is not None + body = json.loads(sent.content) + assert body["aggregates"][0]["function"] == "COUNT" + assert body["aggregates"][0]["alias"] == "total" + assert body["groupBy"] == ["status"] + + def test_query_v2_when_binnings_provided( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + from uipath.platform.entities import EntityAggregateFunction, EntityBinning + + entity_key = uuid.uuid4() + httpx_mock.add_response( + url=re.compile( + rf"{base_url}{org}{tenant}/datafabric_/api/v2/EntityService/entity/{entity_key}/query.*" + ), + status_code=200, + json={"value": [], "totalCount": 0}, + ) + + service.retrieve_records( + entity_key=str(entity_key), + binnings=[ + EntityBinning( + field_name="status", + aggregate_function=EntityAggregateFunction.Count, + alias="total", + ) + ], + ) + + sent = httpx_mock.get_request() + assert sent is not None + assert "/v2/EntityService/" in str(sent.url) + + def test_upload_attachment_sends_multipart( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + entity_id = "ent-1" + record_id = "rec-1" + field_name = "doc" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/Attachment/entity/{entity_id}/{record_id}/{field_name}?expansionLevel=1", + method="POST", + status_code=200, + json={"Id": record_id, "doc": "uploaded"}, + ) + + result = service.upload_attachment( + entity_id=entity_id, + record_id=record_id, + field_name=field_name, + file=b"hello world", + expansion_level=1, + ) + + assert result.get("doc") == "uploaded" + + sent = httpx_mock.get_request() + assert sent is not None + assert b"hello world" in sent.content + + def test_download_attachment_returns_bytes( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + entity_id = "ent-1" + record_id = "rec-1" + field_name = "doc" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/Attachment/entity/{entity_id}/{record_id}/{field_name}", + method="GET", + status_code=200, + content=b"file-content", + ) + + content = service.download_attachment( + entity_id=entity_id, record_id=record_id, field_name=field_name + ) + assert content == b"file-content" + + def test_delete_attachment( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + entity_id = "ent-1" + record_id = "rec-1" + field_name = "doc" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/Attachment/entity/{entity_id}/{record_id}/{field_name}", + method="DELETE", + status_code=200, + json={}, + ) + + result = service.delete_attachment( + entity_id=entity_id, record_id=record_id, field_name=field_name + ) + assert result == {} + + def test_create_entity_returns_id( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + from uipath.platform.entities import ( + EntityCreateFieldOptions, + EntityCreateOptions, + EntityFieldDataType, + ) + + new_entity_id = str(uuid.uuid4()) + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/Entity", + method="POST", + status_code=200, + json=new_entity_id, + ) + + created_id = service.create_entity( + name="productCatalog", + fields=[ + EntityCreateFieldOptions( + field_name="productName", + type=EntityFieldDataType.STRING, + is_required=True, + length_limit=200, + ), + ], + options=EntityCreateOptions( + display_name="Product Catalog", + description="Catalog of products", + is_rbac_enabled=True, + ), + ) + + assert created_id == new_entity_id + sent = httpx_mock.get_request() + assert sent is not None + body = json.loads(sent.content) + assert body["displayName"] == "Product Catalog" + assert body["entityDefinition"]["name"] == "productCatalog" + assert body["entityDefinition"]["fields"][0]["name"] == "productName" + assert body["entityDefinition"]["isRbacEnabled"] is True + + def test_delete_entity( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + entity_id = "ent-doomed" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/Entity/{entity_id}", + method="DELETE", + status_code=200, + ) + + service.delete_entity(entity_id=entity_id) + sent = httpx_mock.get_request() + assert sent is not None + assert sent.method == "DELETE" + + def test_update_entity_metadata( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + from uipath.platform.entities import EntityMetadataUpdateOptions + + entity_id = "ent-meta" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/Entity/{entity_id}/metadata", + method="PATCH", + status_code=200, + json={}, + ) + + service.update_entity_metadata( + entity_id=entity_id, + metadata=EntityMetadataUpdateOptions( + display_name="New Name", is_rbac_enabled=False + ), + ) + + sent = httpx_mock.get_request() + assert sent is not None + body = json.loads(sent.content) + assert body == {"displayName": "New Name", "isRbacEnabled": False} + + def test_import_records( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + entity_id = "ent-imp" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_id}/bulk-upload", + method="POST", + status_code=200, + json={ + "totalRecords": 10, + "insertedRecords": 9, + "errorFileLink": "https://example.com/errors.csv", + }, + ) + + result = service.import_records(entity_id=entity_id, file=b"a,b,c\n1,2,3\n") + assert result.total_records == 10 + assert result.inserted_records == 9 + assert result.error_file_link == "https://example.com/errors.csv" + + def test_list_records_returns_paginated_metadata( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + entity_key = uuid.uuid4() + httpx_mock.add_response( + url=re.compile( + rf"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/read.*" + ), + status_code=200, + json={ + "totalCount": 7, + "value": [{"Id": "1"}, {"Id": "2"}, {"Id": "3"}], + }, + ) + + records = service.list_records( + entity_key=str(entity_key), + start=0, + limit=3, + expansion_level=2, + filter="status eq 'active'", + orderby="name asc", + select=["Id", "name"], + expand=["Company"], + ) + + # New pagination metadata: backend totalCount surfaced verbatim. + assert records.total_count == 7 + assert records.has_next_page is True + # Backend does not currently emit next_cursor; caller paginates with start. + assert records.next_cursor is None + + # Backward-compat: behaves as a list. + assert isinstance(records, list) + assert len(records) == 3 + assert records[0].id == "1" + + sent = httpx_mock.get_request() + assert sent is not None + params = sent.url.params + assert params.get("expansionLevel") == "2" + assert params.get("$filter") == "status eq 'active'" + assert params.get("$orderby") == "name asc" + assert params.get("$select") == "Id,name" + assert params.get("$expand") == "Company" + + def test_insert_records_passes_expansion_level_and_fail_on_first( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + entity_key = uuid.uuid4() + httpx_mock.add_response( + url=re.compile( + rf"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/insert-batch.*" + ), + status_code=200, + json={"successRecords": [{"Id": "1"}], "failureRecords": []}, + ) + + service.insert_records( + entity_key=str(entity_key), + records=[{"name": "alice"}], + expansion_level=1, + fail_on_first=True, + ) + + sent = httpx_mock.get_request() + assert sent is not None + params = sent.url.params + assert params.get("expansionLevel") == "1" + assert params.get("failOnFirst") == "true" + # Records are normalized to dicts before being sent. + assert json.loads(sent.content) == [{"name": "alice"}] + + def test_update_records_recovers_failure_records_from_4xx( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + """A 400 response that lists per-record failures should parse into the response. + + The caller receives an ``EntityRecordsBatchResponse`` with the failed + records populated rather than an exception, so unknown record ids on + update can be handled the same way as any other batch failure. + """ + entity_key = uuid.uuid4() + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/update-batch", + method="POST", + status_code=400, + json={ + "successRecords": [], + "failureRecords": [ + {"error": "Record not found", "record": {"Id": "missing"}} + ], + }, + ) + + result = service.update_records( + entity_key=str(entity_key), + records=[{"Id": "missing", "name": "x"}], + ) + + assert len(result.failure_records) == 1 + assert result.failure_records[0].error == "Record not found" + + def test_delete_records_recovers_failure_records_from_4xx( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + entity_key = uuid.uuid4() + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/delete-batch", + method="POST", + status_code=400, + json={ + "successRecords": [], + "failureRecords": [{"error": "not found"}], + }, + ) + + result = service.delete_records( + entity_key=str(entity_key), record_ids=["missing"] + ) + + assert result.failure_records[0].error == "not found" + + def test_record_to_dict_accepts_dict_pydantic_and_object(self) -> None: + from uipath.platform.entities import EntityCreateFieldOptions + + # dict + assert EntityDataService._record_to_dict({"a": 1}) == {"a": 1} + # Pydantic model — uses model_dump + result = EntityDataService._record_to_dict( + EntityCreateFieldOptions(field_name="x") + ) + assert result["fieldName"] == "x" + # Object with __dict__ + from dataclasses import dataclass + + @dataclass + class Rec: + name: str + + assert EntityDataService._record_to_dict(Rec(name="bob")) == {"name": "bob"} + + +class TestEntitiesServiceCreateEntitySqlTypeMapping: + """Verify ``create_entity`` produces the SQL types and constraint defaults the backend expects.""" + + def _captured_field( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + field_options, + ): + from uipath.platform.entities import EntityCreateOptions + + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/Entity", + method="POST", + status_code=200, + json="00000000-0000-0000-0000-000000000001", + ) + service.create_entity( + name="myEntity", + fields=[field_options], + options=EntityCreateOptions(display_name="My Entity"), + ) + sent = httpx_mock.get_request() + assert sent is not None + body = json.loads(sent.content) + return body["entityDefinition"]["fields"][0] + + def test_string_field_maps_to_nvarchar_with_default_length( + self, httpx_mock, service, base_url, org, tenant, version + ) -> None: + from uipath.platform.entities import ( + EntityCreateFieldOptions, + EntityFieldDataType, + ) + + f = self._captured_field( + httpx_mock, + service, + base_url, + org, + tenant, + EntityCreateFieldOptions( + field_name="productName", type=EntityFieldDataType.STRING + ), + ) + assert f["sqlType"]["name"] == "NVARCHAR" + assert f["sqlType"]["lengthLimit"] == 200 # default + assert f["fieldDisplayType"] == "Basic" + + def test_decimal_field_includes_precision_and_value_bounds( + self, httpx_mock, service, base_url, org, tenant, version + ) -> None: + from uipath.platform.entities import ( + EntityCreateFieldOptions, + EntityFieldDataType, + ) + + f = self._captured_field( + httpx_mock, + service, + base_url, + org, + tenant, + EntityCreateFieldOptions( + field_name="price", + type=EntityFieldDataType.DECIMAL, + decimal_precision=4, + ), + ) + assert f["sqlType"]["name"] == "DECIMAL" + assert f["sqlType"]["decimalPrecision"] == 4 + assert f["sqlType"]["lengthLimit"] == 1000 + assert f["sqlType"]["maxValue"] == 1_000_000_000_000 + assert f["sqlType"]["minValue"] == -1_000_000_000_000 + + def test_boolean_field_maps_to_bit( + self, httpx_mock, service, base_url, org, tenant, version + ) -> None: + from uipath.platform.entities import ( + EntityCreateFieldOptions, + EntityFieldDataType, + ) + + f = self._captured_field( + httpx_mock, + service, + base_url, + org, + tenant, + EntityCreateFieldOptions( + field_name="isActive", type=EntityFieldDataType.BOOLEAN + ), + ) + assert f["sqlType"]["name"] == "BIT" + assert f["sqlType"]["lengthLimit"] == 100 + + def test_file_field_maps_to_uniqueidentifier_with_file_display_type( + self, httpx_mock, service, base_url, org, tenant, version + ) -> None: + from uipath.platform.entities import ( + EntityCreateFieldOptions, + EntityFieldDataType, + ) + + f = self._captured_field( + httpx_mock, + service, + base_url, + org, + tenant, + EntityCreateFieldOptions( + field_name="document", type=EntityFieldDataType.FILE + ), + ) + assert f["sqlType"]["name"] == "UNIQUEIDENTIFIER" + assert f["fieldDisplayType"] == "File" + assert f["sqlType"]["lengthLimit"] == 300 + + +class TestEntitiesServiceValidation: + """Client-side validation rejects bad entity / field definitions before any HTTP call.""" + + def test_create_entity_rejects_invalid_entity_name(self, service) -> None: + + with pytest.raises(ValueError, match="Invalid entity name"): + service.create_entity(name="1bad", fields=[]) + + def test_create_entity_rejects_invalid_field_name(self, service) -> None: + from uipath.platform.entities import EntityCreateFieldOptions + + with pytest.raises(ValueError, match="Invalid field name"): + service.create_entity( + name="goodEntity", + fields=[EntityCreateFieldOptions(field_name="9bad")], + ) + + def test_create_entity_rejects_reserved_field_name(self, service) -> None: + from uipath.platform.entities import EntityCreateFieldOptions + + with pytest.raises(ValueError, match="reserved"): + service.create_entity( + name="goodEntity", + fields=[EntityCreateFieldOptions(field_name="Id")], + ) + + def test_create_entity_rejects_unsupported_constraint_for_type( + self, service + ) -> None: + from uipath.platform.entities import ( + EntityCreateFieldOptions, + EntityFieldDataType, + ) + + with pytest.raises(ValueError, match="does not accept"): + service.create_entity( + name="goodEntity", + fields=[ + EntityCreateFieldOptions( + field_name="myField", + type=EntityFieldDataType.STRING, + decimal_precision=2, # not allowed on STRING + ) + ], + ) + + def test_create_entity_rejects_out_of_range_constraint(self, service) -> None: + from uipath.platform.entities import ( + EntityCreateFieldOptions, + EntityFieldDataType, + ) + + with pytest.raises(ValueError, match="out of range"): + service.create_entity( + name="goodEntity", + fields=[ + EntityCreateFieldOptions( + field_name="myField", + type=EntityFieldDataType.STRING, + length_limit=99999, # > 4000 + ) + ], + ) + + def test_create_entity_rejects_min_ge_max(self, service) -> None: + from uipath.platform.entities import ( + EntityCreateFieldOptions, + EntityFieldDataType, + ) + + with pytest.raises(ValueError, match="strictly less than"): + service.create_entity( + name="goodEntity", + fields=[ + EntityCreateFieldOptions( + field_name="myField", + type=EntityFieldDataType.INTEGER, + min_value=100, + max_value=10, + ) + ], + ) + + def test_create_entity_rejects_choiceset_without_choice_set_id( + self, service + ) -> None: + from uipath.platform.entities import ( + EntityCreateFieldOptions, + EntityFieldDataType, + ) + + with pytest.raises(ValueError, match="choice_set_id"): + service.create_entity( + name="goodEntity", + fields=[ + EntityCreateFieldOptions( + field_name="myField", + type=EntityFieldDataType.CHOICE_SET_SINGLE, + ) + ], + ) + + def test_create_entity_rejects_choice_set_multiple_without_choice_set_id( + self, service + ) -> None: + from uipath.platform.entities import ( + EntityCreateFieldOptions, + EntityFieldDataType, + ) + + with pytest.raises(ValueError, match="choice_set_id"): + service.create_entity( + name="goodEntity", + fields=[ + EntityCreateFieldOptions( + field_name="myField", + type=EntityFieldDataType.CHOICE_SET_MULTIPLE, + ) + ], + ) + + def test_create_entity_rejects_relationship_without_reference_entity( + self, service + ) -> None: + from uipath.platform.entities import ( + EntityCreateFieldOptions, + EntityFieldDataType, + ) + + with pytest.raises(ValueError, match="reference_entity_name"): + service.create_entity( + name="goodEntity", + fields=[ + EntityCreateFieldOptions( + field_name="myField", + type=EntityFieldDataType.RELATIONSHIP, + ) + ], + ) + + def test_entity_query_filter_rejects_in_without_value_list(self) -> None: + from uipath.platform.entities import EntityQueryFilter, QueryFilterOperator + + with pytest.raises(ValueError, match="non-empty value_list"): + EntityQueryFilter( + field_name="status", + operator=QueryFilterOperator.In, + ) + + def test_entity_query_filter_rejects_in_with_scalar_value(self) -> None: + from uipath.platform.entities import EntityQueryFilter, QueryFilterOperator + + with pytest.raises(ValueError, match="value must be omitted"): + EntityQueryFilter( + field_name="status", + operator=QueryFilterOperator.In, + value="active", + value_list=["active", "pending"], + ) + + def test_entity_query_filter_rejects_scalar_op_with_value_list(self) -> None: + from uipath.platform.entities import EntityQueryFilter, QueryFilterOperator + + with pytest.raises(ValueError, match="value_list must be omitted"): + EntityQueryFilter( + field_name="amount", + operator=QueryFilterOperator.GreaterThan, + value_list=["10"], + ) + + def test_entity_query_filter_rejects_strict_op_with_null_value(self) -> None: + from uipath.platform.entities import EntityQueryFilter, QueryFilterOperator + + with pytest.raises(ValueError, match="non-null value"): + EntityQueryFilter( + field_name="amount", + operator=QueryFilterOperator.GreaterThan, + ) + + def test_entity_query_filter_allows_equals_with_null_value(self) -> None: + from uipath.platform.entities import EntityQueryFilter, QueryFilterOperator + + f = EntityQueryFilter( + field_name="middle_name", + operator=QueryFilterOperator.Equals, + ) + assert f.value is None and f.value_list is None + + def test_entity_query_filter_allows_in_with_value_list(self) -> None: + from uipath.platform.entities import EntityQueryFilter, QueryFilterOperator + + f = EntityQueryFilter( + field_name="status", + operator=QueryFilterOperator.In, + value_list=["a", "b"], + ) + assert f.value_list == ["a", "b"] + + +class TestEntitiesServiceAsyncAndEdgeCases: + async def test_get_record_async( + self, httpx_mock, service, base_url, org, tenant, version + ) -> None: + entity_key = uuid.uuid4() + record_id = "rec-1" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/read/{record_id}", + status_code=200, + json={"Id": record_id, "name": "found"}, + ) + record = await service.get_record_async( + entity_key=str(entity_key), record_id=record_id + ) + assert record.id == record_id + + async def test_query_async_v1( + self, httpx_mock, service, base_url, org, tenant, version + ) -> None: + entity_key = uuid.uuid4() + httpx_mock.add_response( + url=re.compile( + rf"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/query" + ), + status_code=200, + json={"value": [{"Id": "1"}], "totalRecordCount": 1}, + ) + result = await service.retrieve_records_async(entity_key=str(entity_key)) + assert result.total_count == 1 + + async def test_delete_record_async( + self, httpx_mock, service, base_url, org, tenant, version + ) -> None: + entity_key = uuid.uuid4() + record_id = "rec-1" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/delete/{record_id}", + method="DELETE", + status_code=200, + ) + await service.delete_record_async( + entity_key=str(entity_key), record_id=record_id + ) + + async def test_create_entity_async( + self, httpx_mock, service, base_url, org, tenant, version + ) -> None: + from uipath.platform.entities import ( + EntityCreateFieldOptions, + EntityFieldDataType, + ) + + new_id = "00000000-0000-0000-0000-000000000123" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/Entity", + method="POST", + status_code=200, + json=new_id, + ) + result = await service.create_entity_async( + name="goodEntity", + fields=[ + EntityCreateFieldOptions( + field_name="myField", type=EntityFieldDataType.STRING + ) + ], + ) + assert result == new_id + + async def test_delete_entity_async( + self, httpx_mock, service, base_url, org, tenant, version + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/Entity/ent-1", + method="DELETE", + status_code=200, + ) + await service.delete_entity_async(entity_id="ent-1") + + async def test_update_entity_metadata_async_with_dict( + self, httpx_mock, service, base_url, org, tenant, version + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/Entity/ent-1/metadata", + method="PATCH", + status_code=200, + json={}, + ) + # Accepts a plain dict too + await service.update_entity_metadata_async( + entity_id="ent-1", metadata={"displayName": "X", "description": "Y"} + ) + sent = httpx_mock.get_request() + assert sent is not None + assert json.loads(sent.content) == {"displayName": "X", "description": "Y"} + + def test_update_entity_metadata_normalizes_snake_case_dict_keys( + self, httpx_mock, service, base_url, org, tenant, version + ) -> None: + """Snake_case dict keys must be sent to the backend as camelCase.""" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/Entity/ent-1/metadata", + method="PATCH", + status_code=200, + json={}, + ) + service.update_entity_metadata( + entity_id="ent-1", + metadata={ + "display_name": "New Name", + "description": "Updated", + "is_rbac_enabled": True, + }, + ) + sent = httpx_mock.get_request() + assert sent is not None + assert json.loads(sent.content) == { + "displayName": "New Name", + "description": "Updated", + "isRbacEnabled": True, + } + + async def test_upload_attachment_async_via_file_path( + self, httpx_mock, service, base_url, org, tenant, version, tmp_path + ) -> None: + path = tmp_path / "data.bin" + path.write_bytes(b"file-on-disk") + + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/Attachment/entity/ent/rec/doc", + method="POST", + status_code=200, + json={"Id": "rec", "doc": "ok"}, + ) + result = await service.upload_attachment_async( + entity_id="ent", + record_id="rec", + field_name="doc", + file_path=str(path), + ) + assert result["doc"] == "ok" + + sent = httpx_mock.get_request() + assert sent is not None + assert b"file-on-disk" in sent.content + + async def test_download_and_delete_attachment_async( + self, httpx_mock, service, base_url, org, tenant, version + ) -> None: + url = f"{base_url}{org}{tenant}/datafabric_/api/Attachment/entity/e/r/f" + httpx_mock.add_response( + url=url, method="GET", status_code=200, content=b"bytes" + ) + httpx_mock.add_response(url=url, method="DELETE", status_code=200, json={}) + + content = await service.download_attachment_async( + entity_id="e", record_id="r", field_name="f" + ) + assert content == b"bytes" + assert ( + await service.delete_attachment_async( + entity_id="e", record_id="r", field_name="f" + ) + == {} + ) + + def test_open_file_rejects_both_file_and_path(self) -> None: + with pytest.raises(ValueError, match="exactly one of"): + EntityDataService._open_file(file=b"x", file_path="some/path") + + def test_open_file_rejects_neither_file_nor_path(self) -> None: + with pytest.raises(ValueError, match="exactly one of"): + EntityDataService._open_file(file=None, file_path=None) + + def test_4xx_recovery_only_400_with_strict_shape( + self, httpx_mock, service, base_url, org, tenant, version + ) -> None: + """5xx and 4xx other than 400 must propagate; 400 with valid shape recovers.""" + entity_key = uuid.uuid4() + # 500 with the shape — must propagate, not be silently treated as success. + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/update-batch", + method="POST", + status_code=500, + json={"successRecords": [], "failureRecords": []}, + ) + from uipath.platform.errors._enriched_exception import EnrichedException + + with pytest.raises(EnrichedException): + service.update_records( + entity_key=str(entity_key), records=[{"Id": "x", "name": "y"}] + ) + + def test_4xx_recovery_404_propagates( + self, httpx_mock, service, base_url, org, tenant, version + ) -> None: + entity_key = uuid.uuid4() + # 404 with valid shape — still propagates because not a 400. + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/update-batch", + method="POST", + status_code=404, + json={"successRecords": [], "failureRecords": []}, + ) + from uipath.platform.errors._enriched_exception import EnrichedException + + with pytest.raises(EnrichedException): + service.update_records( + entity_key=str(entity_key), records=[{"Id": "x", "name": "y"}] + ) + + def test_4xx_recovery_400_unrelated_body_propagates( + self, httpx_mock, service, base_url, org, tenant, version + ) -> None: + """A 400 with an error body that lacks ``successRecords``/``failureRecords`` + must surface as an exception (so generic validation errors aren't masked).""" + entity_key = uuid.uuid4() + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/update-batch", + method="POST", + status_code=400, + json={"error": "Validation failed", "code": "InvalidArg"}, + ) + from uipath.platform.errors._enriched_exception import EnrichedException + + with pytest.raises(EnrichedException): + service.update_records( + entity_key=str(entity_key), records=[{"Id": "x", "name": "y"}] + ) + + +class TestEntitiesServiceAsyncCoverage: + """Async-variant tests for previously uncovered paths on schema / data services.""" + + async def test_retrieve_async( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + entity_key = uuid.uuid4() + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/Entity/{entity_key}", + status_code=200, + json={ + "name": "Customers", + "displayName": "Customers", + "entityType": "Entity", + "fields": [], + "isRbacEnabled": False, + "id": str(entity_key), + }, + ) + entity = await service.retrieve_async(entity_key=str(entity_key)) + assert entity.id == str(entity_key) + + def test_retrieve_by_name( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/Entity/Customers/metadata", + status_code=200, + json={ + "name": "Customers", + "displayName": "Customers", + "entityType": "Entity", + "fields": [], + "isRbacEnabled": False, + "id": "ent-1", + }, + ) + entity = service.retrieve_by_name("Customers", folder_key="folder-1") + assert entity.name == "Customers" + sent = httpx_mock.get_request() + assert sent is not None + assert sent.headers.get("X-UIPATH-FolderKey") == "folder-1" + + async def test_retrieve_by_name_async( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/Entity/Orders/metadata", + status_code=200, + json={ + "name": "Orders", + "displayName": "Orders", + "entityType": "Entity", + "fields": [], + "isRbacEnabled": False, + "id": "ent-2", + }, + ) + entity = await service.retrieve_by_name_async("Orders") + assert entity.name == "Orders" + + def test_list_entities_basic( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/Entity", + status_code=200, + json=[ + { + "name": "Customers", + "displayName": "Customers", + "entityType": "Entity", + "fields": [], + "isRbacEnabled": False, + "id": "ent-1", + }, + { + "name": "Orders", + "displayName": "Orders", + "entityType": "Entity", + "fields": [], + "isRbacEnabled": False, + "id": "ent-2", + }, + ], + ) + entities = service.list_entities() + assert len(entities) == 2 + + async def test_list_entities_async( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/Entity", + status_code=200, + json=[ + { + "name": "Customers", + "displayName": "Customers", + "entityType": "Entity", + "fields": [], + "isRbacEnabled": False, + "id": "ent-1", + } + ], + ) + entities = await service.list_entities_async() + assert len(entities) == 1 + + async def test_list_records_async( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + entity_key = uuid.uuid4() + httpx_mock.add_response( + url=re.compile( + rf"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/read.*" + ), + status_code=200, + json={ + "totalRecordCount": 2, + "value": [{"Id": "1"}, {"Id": "2"}], + }, + ) + records = await service.list_records_async( + entity_key=str(entity_key), start=0, limit=10 + ) + assert records.total_count == 2 + + async def test_update_record_async( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + entity_key = uuid.uuid4() + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/update/rec-1", + method="POST", + status_code=200, + json={"Id": "rec-1", "name": "renamed"}, + ) + rec = await service.update_record_async( + entity_key=str(entity_key), + record_id="rec-1", + data={"name": "renamed"}, + ) + assert rec.id == "rec-1" + + async def test_insert_records_async_batch( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + entity_key = uuid.uuid4() + httpx_mock.add_response( + url=re.compile( + rf"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/insert-batch.*" + ), + status_code=200, + json={ + "successRecords": [{"Id": "1", "name": "a"}], + "failureRecords": [], + }, + ) + result = await service.insert_records_async( + entity_key=str(entity_key), + records=[{"name": "a"}], + expansion_level=1, + fail_on_first=True, + ) + assert len(result.success_records) == 1 + + async def test_update_records_async_recovers_400_failures( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + entity_key = uuid.uuid4() + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/update-batch", + method="POST", + status_code=400, + json={ + "successRecords": [], + "failureRecords": [{"error": "not found"}], + }, + ) + result = await service.update_records_async( + entity_key=str(entity_key), + records=[{"Id": "missing", "name": "x"}], + ) + assert result.failure_records[0].error == "not found" + + async def test_delete_records_async_batch( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + entity_key = uuid.uuid4() + httpx_mock.add_response( + url=re.compile( + rf"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/delete-batch.*" + ), + status_code=200, + json={ + "successRecords": [{"Id": "rec-1"}], + "failureRecords": [], + }, + ) + result = await service.delete_records_async( + entity_key=str(entity_key), + record_ids=["rec-1"], + fail_on_first=False, + ) + assert len(result.success_records) == 1 + + async def test_import_records_async( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/ent-1/bulk-upload", + method="POST", + status_code=200, + json={ + "totalRecords": 3, + "insertedRecords": 3, + "errorFileLink": None, + }, + ) + result = await service.import_records_async( + entity_id="ent-1", file=b"a,b\n1,2\n" + ) + assert result.inserted_records == 3 + + def test_validate_entity_batch_handles_success_and_failure_records( + self, + service: EntitiesService, + ) -> None: + response = MagicMock() + response.json.return_value = { + "successRecords": [{"Id": "ok-1", "name": "first"}], + "failureRecords": [{"error": "duplicate", "record": {"name": "dup"}}], + } + result = service.validate_entity_batch(response) + assert len(result.success_records) == 1 + assert result.success_records[0].id == "ok-1" + assert result.failure_records[0].error == "duplicate" + + def test_5xx_with_batch_shape_still_propagates( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + """500 with successRecords/failureRecords shape must NOT be recovered.""" + from uipath.platform.errors._enriched_exception import EnrichedException + + entity_key = uuid.uuid4() + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/EntityService/entity/{entity_key}/insert-batch", + method="POST", + status_code=500, + json={"successRecords": [], "failureRecords": []}, + ) + with pytest.raises(EnrichedException): + service.insert_records( + entity_key=str(entity_key), + records=[{"name": "x"}], + ) + + +class TestGetOntologyFileAsync: + """Tests for EntitiesService.get_ontology_file_async (delegates to + EntityOntologyService). The HTTP call goes through ``service._ontology``, + so the sub-service's ``request_async`` is what gets patched.""" + + @pytest.mark.anyio + async def test_builds_endpoint_and_folder_header( + self, service: EntitiesService + ) -> None: + response = MagicMock() + response.json.return_value = {"content": "OWL", "mediaType": "text/plain"} + service._ontology.request_async = AsyncMock(return_value=response) # type: ignore[method-assign] + + result = await service.get_ontology_file_async( + "library", "owl", folder_key="folder-1" + ) + + assert result == {"content": "OWL", "mediaType": "text/plain"} + service._ontology.request_async.assert_called_once() + call = service._ontology.request_async.call_args + method, endpoint = call.args[0], call.args[1] + headers = call.kwargs["headers"] + assert method == "GET" + assert str(endpoint) == "/datafabric_/api/ontologies/library/files/owl" + # Accept is added centrally by BaseService, not per-call. + assert headers["x-uipath-folderkey"] == "folder-1" + + @pytest.mark.anyio + async def test_no_folder_header_when_folder_key_none( + self, service: EntitiesService + ) -> None: + response = MagicMock() + response.json.return_value = {"content": "OWL", "mediaType": "text/plain"} + service._ontology.request_async = AsyncMock(return_value=response) # type: ignore[method-assign] + + await service.get_ontology_file_async("library") + + headers = service._ontology.request_async.call_args.kwargs["headers"] + assert "x-uipath-folderkey" not in headers + + @pytest.mark.anyio + @pytest.mark.parametrize( + "file_type", ["owl", "r2rml", "shacl", "summary", "context"] + ) + async def test_accepts_allowed_file_types( + self, service: EntitiesService, file_type: str + ) -> None: + response = MagicMock() + response.json.return_value = {"content": "x"} + service._ontology.request_async = AsyncMock(return_value=response) # type: ignore[method-assign] + + await service.get_ontology_file_async("library", file_type) + + endpoint = service._ontology.request_async.call_args.args[1] + assert str(endpoint) == f"/datafabric_/api/ontologies/library/files/{file_type}" + + @pytest.mark.anyio + async def test_rejects_unsupported_file_type( + self, + httpx_mock: HTTPXMock, + service: EntitiesService, + base_url: str, + org: str, + tenant: str, + version: str, + ) -> None: + """File-type validation is server-side (the ontology API), not in the + client. The SDK forwards the requested type and must surface the API's + rejection of an unsupported one as ``EnrichedException`` rather than + swallowing it.""" + api_message = "Unsupported ontology file type: exe" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/datafabric_/api/ontologies/library/files/exe", + method="GET", + status_code=400, + json={"message": api_message}, + ) + + with pytest.raises(EnrichedException) as exc_info: + await service.get_ontology_file_async("library", "exe") + + # The SDK surfaces the API's rejection verbatim — status code, response + # body, and the extracted message — rather than masking it. + exc = exc_info.value + assert exc.status_code == 400 + assert api_message in exc.response_content + assert exc.error_info is not None + assert exc.error_info.message == api_message diff --git a/packages/uipath-platform/tests/services/test_external_application_service.py b/packages/uipath-platform/tests/services/test_external_application_service.py index 15c267888..1fca91b32 100644 --- a/packages/uipath-platform/tests/services/test_external_application_service.py +++ b/packages/uipath-platform/tests/services/test_external_application_service.py @@ -3,11 +3,9 @@ import httpx import pytest -from uipath.platform.common._external_application_service import ( - ExternalApplicationService, -) from uipath.platform.common.auth import TokenData from uipath.platform.errors import EnrichedException +from uipath.platform.external_applications import ExternalApplicationService IDENTITY_SERVICE_SYNC = ( "uipath.platform.identity.IdentityService.get_client_credentials_token" diff --git a/packages/uipath-platform/tests/services/test_folder_context.py b/packages/uipath-platform/tests/services/test_folder_context.py index ceaf33f88..e2fcaf6ca 100644 --- a/packages/uipath-platform/tests/services/test_folder_context.py +++ b/packages/uipath-platform/tests/services/test_folder_context.py @@ -15,7 +15,7 @@ folder_path_header, header_folder, ) -from uipath.platform.common.constants import ( +from uipath.platform.constants import ( HEADER_FOLDER_KEY, HEADER_FOLDER_PATH, HEADER_FOLDER_PATH_ENCODED, diff --git a/packages/uipath-platform/tests/services/test_folder_service.py b/packages/uipath-platform/tests/services/test_folder_service.py index ae4b6804f..023d85ae8 100644 --- a/packages/uipath-platform/tests/services/test_folder_service.py +++ b/packages/uipath-platform/tests/services/test_folder_service.py @@ -2,7 +2,7 @@ from pytest_httpx import HTTPXMock from uipath.platform import UiPathApiConfig, UiPathExecutionContext -from uipath.platform.common.constants import HEADER_USER_AGENT +from uipath.platform.constants import HEADER_USER_AGENT from uipath.platform.errors import FolderNotFoundException from uipath.platform.orchestrator._folder_service import FolderService diff --git a/packages/uipath-platform/tests/services/test_governance_provider.py b/packages/uipath-platform/tests/services/test_governance_provider.py new file mode 100644 index 000000000..f7211d97a --- /dev/null +++ b/packages/uipath-platform/tests/services/test_governance_provider.py @@ -0,0 +1,202 @@ +"""Tests for UiPathPlatformGovernanceProvider.""" + +from __future__ import annotations + +import pytest +from pytest_httpx import HTTPXMock +from uipath.core.governance import ( + EnforcementMode, + FiredRule, + GovernanceCompensationProvider, + GovernancePolicyProvider, + GovernRequest, + PolicyContext, +) + +from uipath.platform import UiPathApiConfig, UiPathExecutionContext +from uipath.platform.governance import ( + GovernanceService, + UiPathPlatformGovernanceProvider, +) + +ORG_ID = "11111111-1111-1111-1111-111111111111" +TENANT_ID = "22222222-2222-2222-2222-222222222222" + + +def _make_request() -> GovernRequest: + return GovernRequest( + validators=["pii_detection"], + rules=[ + FiredRule( + rule_id="ASI-01", + rule_name="Block PII in flight", + pack_name="agent-safety", + validator="pii_detection", + ) + ], + data={"prompt": "hello"}, + hook="before_model", + trace_id="0123456789abcdef0123456789abcdef", + src_timestamp="2026-06-22T10:00:00Z", + agent_name="my-agent", + runtime_id="runtime-1", + ) + + +@pytest.fixture +def provider( + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + monkeypatch: pytest.MonkeyPatch, +) -> UiPathPlatformGovernanceProvider: + monkeypatch.setenv("UIPATH_ORGANIZATION_ID", ORG_ID) + monkeypatch.setenv("UIPATH_TENANT_ID", TENANT_ID) + service = GovernanceService(config=config, execution_context=execution_context) + return UiPathPlatformGovernanceProvider(service=service) + + +class TestConstruction: + def test_accepts_existing_service( + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + ) -> None: + service = GovernanceService(config=config, execution_context=execution_context) + provider = UiPathPlatformGovernanceProvider(service=service) + assert provider.service is service + + def test_builds_service_from_config( + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + ) -> None: + provider = UiPathPlatformGovernanceProvider( + config=config, execution_context=execution_context + ) + assert isinstance(provider.service, GovernanceService) + + def test_requires_service_or_full_kwargs(self) -> None: + with pytest.raises(ValueError, match="GovernanceService"): + UiPathPlatformGovernanceProvider() + + +class TestProtocolConformance: + def test_satisfies_policy_provider_protocol( + self, provider: UiPathPlatformGovernanceProvider + ) -> None: + assert isinstance(provider, GovernancePolicyProvider) + + def test_satisfies_compensation_provider_protocol( + self, provider: UiPathPlatformGovernanceProvider + ) -> None: + assert isinstance(provider, GovernanceCompensationProvider) + + +class TestDelegation: + def test_get_policy_delegates_to_service( + self, + httpx_mock: HTTPXMock, + provider: UiPathPlatformGovernanceProvider, + base_url: str, + ) -> None: + httpx_mock.add_response( + url=( + f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/policy" + "?agentType=conversational" + ), + status_code=200, + json={"mode": "enforce", "policies": "rules: []"}, + ) + + response = provider.get_policy(PolicyContext(is_conversational=True)) + + assert response.mode is EnforcementMode.ENFORCE + assert response.policies == "rules: []" + + async def test_get_policy_async_delegates_to_service( + self, + httpx_mock: HTTPXMock, + provider: UiPathPlatformGovernanceProvider, + base_url: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/policy", + status_code=200, + json={"mode": "audit", "policies": ""}, + ) + + response = await provider.get_policy_async(PolicyContext()) + + assert response.mode is EnforcementMode.AUDIT + + def test_compensate_delegates_to_service( + self, + httpx_mock: HTTPXMock, + provider: UiPathPlatformGovernanceProvider, + base_url: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/govern", + status_code=200, + json={}, + ) + + provider.compensate(_make_request()) + + requests = httpx_mock.get_requests() + assert len(requests) == 1 + assert requests[0].method == "POST" + + async def test_compensate_async_delegates_to_service( + self, + httpx_mock: HTTPXMock, + provider: UiPathPlatformGovernanceProvider, + base_url: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/govern", + status_code=200, + json={}, + ) + + await provider.compensate_async(_make_request()) + + requests = httpx_mock.get_requests() + assert len(requests) == 1 + assert requests[0].method == "POST" + + def test_track_event_delegates_to_service( + self, + httpx_mock: HTTPXMock, + provider: UiPathPlatformGovernanceProvider, + base_url: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/log", + method="POST", + status_code=204, + ) + + provider.track_event(event_name="ev", data={"k": "v"}, operation_id="op-1") + + sent = httpx_mock.get_requests()[-1] + assert sent.method == "POST" + assert sent.headers["x-uipath-operation-id"] == "op-1" + + async def test_track_event_async_delegates_to_service( + self, + httpx_mock: HTTPXMock, + provider: UiPathPlatformGovernanceProvider, + base_url: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/log", + method="POST", + status_code=204, + ) + + await provider.track_event_async(event_name="ev", operation_id="op-2") + + sent = httpx_mock.get_requests()[-1] + assert sent.method == "POST" + assert sent.headers["x-uipath-operation-id"] == "op-2" diff --git a/packages/uipath-platform/tests/services/test_governance_service.py b/packages/uipath-platform/tests/services/test_governance_service.py new file mode 100644 index 000000000..eb4941faf --- /dev/null +++ b/packages/uipath-platform/tests/services/test_governance_service.py @@ -0,0 +1,934 @@ +"""Tests for GovernanceService.""" + +from __future__ import annotations + +import json +from typing import Any + +import httpx +import pytest +from pytest_httpx import HTTPXMock +from uipath.core.governance import GovernancePolicyProvider, PolicyContext + +from uipath.platform import UiPathApiConfig, UiPathExecutionContext +from uipath.platform.common import resolve_trace_id +from uipath.platform.governance import ( + FiredRule, + GovernanceService, + PolicyResponse, +) + +ORG_ID = "11111111-1111-1111-1111-111111111111" +TENANT_ID = "22222222-2222-2222-2222-222222222222" +TENANT_ID_HEX = TENANT_ID.replace("-", "").lower() + + +def _compensate_kwargs(**overrides: Any) -> dict[str, Any]: + """Default kwargs for ``service.compensate(...)``.""" + defaults: dict[str, Any] = dict( + validators=["pii_detection"], + rules=[ + FiredRule( + rule_id="ASI-01", + rule_name="Block PII in flight", + pack_name="agent-safety", + validator="pii_detection", + ) + ], + data={"prompt": "hello"}, + hook="before_model", + trace_id="0123456789abcdef0123456789abcdef", + src_timestamp="2026-06-22T10:00:00Z", + agent_name="my-agent", + runtime_id="runtime-1", + ) + defaults.update(overrides) + return defaults + + +@pytest.fixture +def service( + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + monkeypatch: pytest.MonkeyPatch, +) -> GovernanceService: + monkeypatch.setenv("UIPATH_ORGANIZATION_ID", ORG_ID) + monkeypatch.setenv("UIPATH_TENANT_ID", TENANT_ID) + return GovernanceService(config=config, execution_context=execution_context) + + +class TestGovernanceService: + """Test GovernanceService functionality.""" + + class TestRetrievePolicy: + """Test retrieve_policy (sync).""" + + def test_returns_parsed_policy( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/policy", + status_code=200, + json={"mode": "enforce", "policies": "rules: []"}, + ) + + result = service.retrieve_policy() + + assert isinstance(result, PolicyResponse) + assert result.mode == "enforce" + assert result.policies == "rules: []" + + def test_defaults_when_fields_missing( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/policy", + status_code=200, + json={}, + ) + + result = service.retrieve_policy() + + assert result.mode is None + assert result.policies == "" + + def test_sends_tenant_header_and_bearer_token( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + secret: str, + ) -> None: + captured: dict[str, httpx.Request] = {} + + def capture(request: httpx.Request) -> httpx.Response: + captured["request"] = request + return httpx.Response(200, json={"mode": "audit", "policies": ""}) + + httpx_mock.add_callback( + capture, + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/policy", + ) + + service.retrieve_policy() + + request = captured["request"] + assert request.method == "GET" + assert request.headers["x-uipath-internal-tenantid"] == TENANT_ID + assert request.headers["authorization"] == f"Bearer {secret}" + # No agentType query param when caller omits it. + assert "agentType" not in request.url.params + + @pytest.mark.parametrize( + ("is_conversational", "expected"), + [(True, "conversational"), (False, "autonomous")], + ) + def test_appends_agent_type_query_param( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + is_conversational: bool, + expected: str, + ) -> None: + captured: dict[str, httpx.Request] = {} + + def capture(request: httpx.Request) -> httpx.Response: + captured["request"] = request + return httpx.Response(200, json={"mode": "audit", "policies": ""}) + + httpx_mock.add_callback( + capture, + url=( + f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/policy" + f"?agentType={expected}" + ), + ) + + service.retrieve_policy(is_conversational=is_conversational) + + assert captured["request"].url.params["agentType"] == expected + + def test_raises_when_organization_id_missing( + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.delenv("UIPATH_ORGANIZATION_ID", raising=False) + monkeypatch.setenv("UIPATH_TENANT_ID", TENANT_ID) + service = GovernanceService( + config=config, execution_context=execution_context + ) + + with pytest.raises(ValueError, match="UIPATH_ORGANIZATION_ID"): + service.retrieve_policy() + + def test_raises_when_tenant_id_missing( + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv("UIPATH_ORGANIZATION_ID", ORG_ID) + monkeypatch.delenv("UIPATH_TENANT_ID", raising=False) + service = GovernanceService( + config=config, execution_context=execution_context + ) + + with pytest.raises(ValueError, match="UIPATH_TENANT_ID"): + service.retrieve_policy() + + def test_raises_on_http_error( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + ) -> None: + from uipath.platform.errors import EnrichedException + + httpx_mock.add_response( + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/policy", + status_code=500, + text="boom", + ) + + with pytest.raises(EnrichedException): + service.retrieve_policy() + + class TestRetrievePolicyAsync: + """Test retrieve_policy_async.""" + + async def test_returns_parsed_policy( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + ) -> None: + httpx_mock.add_response( + url=( + f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/policy" + "?agentType=autonomous" + ), + status_code=200, + json={"mode": "audit", "policies": "rules: []"}, + ) + + result = await service.retrieve_policy_async(is_conversational=False) + + assert result.mode == "audit" + assert result.policies == "rules: []" + + class TestCompensate: + """Test compensate (sync).""" + + def test_posts_aliased_payload( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + secret: str, + ) -> None: + captured: dict[str, httpx.Request] = {} + + def capture(request: httpx.Request) -> httpx.Response: + captured["request"] = request + return httpx.Response(200, json={}) + + httpx_mock.add_callback( + capture, + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/govern", + ) + + service.compensate(**_compensate_kwargs()) + + request = captured["request"] + assert request.method == "POST" + assert request.headers["x-uipath-internal-tenantid"] == TENANT_ID + assert request.headers["authorization"] == f"Bearer {secret}" + + body = json.loads(request.content) + assert body["type"] == ["pii_detection"] + assert body["rules"] == [ + { + "ruleId": "ASI-01", + "ruleName": "Block PII in flight", + "packName": "agent-safety", + "validator": "pii_detection", + } + ] + assert body["traceId"] == "0123456789abcdef0123456789abcdef" + assert body["src_timestamp"] == "2026-06-22T10:00:00Z" + assert body["agentName"] == "my-agent" + assert body["runtimeId"] == "runtime-1" + assert body["hook"] == "before_model" + assert body["data"] == {"prompt": "hello"} + + def test_autofills_job_context_from_config( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv("UIPATH_FOLDER_KEY", "folder-from-env") + monkeypatch.setenv("UIPATH_JOB_KEY", "job-from-env") + monkeypatch.setenv("UIPATH_PROCESS_UUID", "process-from-env") + monkeypatch.setenv("UIPATH_AGENT_ID", "agent-from-env") + monkeypatch.setenv("UIPATH_PROCESS_VERSION", "1.2.3") + + captured: dict[str, httpx.Request] = {} + + def capture(request: httpx.Request) -> httpx.Response: + captured["request"] = request + return httpx.Response(200, json={}) + + httpx_mock.add_callback( + capture, + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/govern", + ) + + service.compensate(**_compensate_kwargs()) + + body = json.loads(captured["request"].content) + assert body["folderKey"] == "folder-from-env" + assert body["jobKey"] == "job-from-env" + assert body["processKey"] == "process-from-env" + assert body["referenceId"] == "agent-from-env" + assert body["agentVersion"] == "1.2.3" + + def test_caller_overrides_take_precedence( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv("UIPATH_FOLDER_KEY", "env-folder") + monkeypatch.setenv("UIPATH_JOB_KEY", "env-job") + + captured: dict[str, httpx.Request] = {} + + def capture(request: httpx.Request) -> httpx.Response: + captured["request"] = request + return httpx.Response(200, json={}) + + httpx_mock.add_callback( + capture, + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/govern", + ) + + service.compensate(**_compensate_kwargs(folder_key="explicit-folder")) + + body = json.loads(captured["request"].content) + # Caller-supplied value wins. + assert body["folderKey"] == "explicit-folder" + # Env-backed fallback fills the unset one. + assert body["jobKey"] == "env-job" + # Unset and unbacked → key omitted. + assert "processKey" not in body + + def test_caller_empty_string_is_not_overridden_by_env( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv("UIPATH_FOLDER_KEY", "env-folder") + + captured: dict[str, httpx.Request] = {} + + def capture(request: httpx.Request) -> httpx.Response: + captured["request"] = request + return httpx.Response(200, json={}) + + httpx_mock.add_callback( + capture, + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/govern", + ) + + # Explicit empty string is still a caller value — must not be + # silently replaced by the env-backed UiPathConfig fallback. + service.compensate(**_compensate_kwargs(folder_key="")) + + body = json.loads(captured["request"].content) + assert body["folderKey"] == "" + + def test_omits_job_context_keys_with_no_value( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + for env_key in ( + "UIPATH_FOLDER_KEY", + "UIPATH_JOB_KEY", + "UIPATH_PROCESS_UUID", + "UIPATH_AGENT_ID", + "UIPATH_PROCESS_VERSION", + "UIPATH_PROJECT_ID", + ): + monkeypatch.delenv(env_key, raising=False) + + captured: dict[str, httpx.Request] = {} + + def capture(request: httpx.Request) -> httpx.Response: + captured["request"] = request + return httpx.Response(200, json={}) + + httpx_mock.add_callback( + capture, + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/govern", + ) + + service.compensate(**_compensate_kwargs()) + + body = json.loads(captured["request"].content) + for absent in ( + "folderKey", + "jobKey", + "processKey", + "referenceId", + "agentVersion", + ): + assert absent not in body + + def test_raises_on_http_error( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + ) -> None: + from uipath.platform.errors import EnrichedException + + httpx_mock.add_response( + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/govern", + status_code=400, + text="bad payload", + ) + + with pytest.raises(EnrichedException): + service.compensate(**_compensate_kwargs()) + + def test_raises_when_org_id_missing( + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.delenv("UIPATH_ORGANIZATION_ID", raising=False) + monkeypatch.setenv("UIPATH_TENANT_ID", TENANT_ID) + service = GovernanceService( + config=config, execution_context=execution_context + ) + + with pytest.raises(ValueError, match="UIPATH_ORGANIZATION_ID"): + service.compensate(**_compensate_kwargs()) + + def test_self_resolves_trace_id_when_caller_leaves_none( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """``trace_id=None`` from the caller is filled via resolve_trace_id(). + + The runtime layer intentionally stays env-free; the platform + service fills the canonical trace id at HTTP-call time from + the OTel/env source. ``UIPATH_TRACE_ID`` covers the + resolver-finds-a-value branch of ``_resolve_request_trace_id``. + """ + monkeypatch.setenv("UIPATH_TRACE_ID", TENANT_ID_HEX) + captured: dict[str, httpx.Request] = {} + + def capture(request: httpx.Request) -> httpx.Response: + captured["request"] = request + return httpx.Response(200, json={}) + + httpx_mock.add_callback( + capture, + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/govern", + ) + + service.compensate(**_compensate_kwargs(trace_id=None)) + + body = json.loads(captured["request"].content) + assert body["traceId"] == TENANT_ID_HEX + + def test_omits_trace_id_when_no_source_resolves( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Resolver returns nothing → traceId is omitted from the body. + + Covers the resolver-finds-nothing branch of + ``_resolve_request_trace_id``: no ``UIPATH_TRACE_ID``, no + active OTel context → ``trace_id`` stays ``None`` on the + request and ``model_dump(exclude_none=True)`` drops it from + the wire JSON. + """ + monkeypatch.delenv("UIPATH_TRACE_ID", raising=False) + captured: dict[str, httpx.Request] = {} + + def capture(request: httpx.Request) -> httpx.Response: + captured["request"] = request + return httpx.Response(200, json={}) + + httpx_mock.add_callback( + capture, + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/govern", + ) + + service.compensate(**_compensate_kwargs(trace_id=None)) + + body = json.loads(captured["request"].content) + assert "traceId" not in body + + def test_caller_empty_string_wins_over_resolver( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """An explicit ``trace_id=""`` from the caller is not overridden. + + With the absence-via-``None`` contract, the empty string is + a legitimate caller-supplied value — it must not trigger + the auto-resolve. + """ + monkeypatch.setenv("UIPATH_TRACE_ID", TENANT_ID_HEX) + captured: dict[str, httpx.Request] = {} + + def capture(request: httpx.Request) -> httpx.Response: + captured["request"] = request + return httpx.Response(200, json={}) + + httpx_mock.add_callback( + capture, + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/govern", + ) + + service.compensate(**_compensate_kwargs(trace_id="")) + + body = json.loads(captured["request"].content) + assert body["traceId"] == "" + + class TestCompensateAsync: + """Test compensate_async.""" + + async def test_posts_aliased_payload( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/govern", + status_code=200, + json={}, + ) + + await service.compensate_async(**_compensate_kwargs()) + + requests = httpx_mock.get_requests() + assert len(requests) == 1 + assert requests[0].method == "POST" + + class TestProtocolConformance: + """``get_policy`` adapter is the only protocol-shaped surface left + on :class:`GovernanceService`; compensation conformance is tested + against :class:`UiPathPlatformGovernanceProvider`. + """ + + def test_satisfies_policy_provider_protocol( + self, service: GovernanceService + ) -> None: + assert isinstance(service, GovernancePolicyProvider) + + def test_get_policy_delegates_to_retrieve_policy( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + ) -> None: + httpx_mock.add_response( + url=( + f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/policy" + "?agentType=conversational" + ), + status_code=200, + json={"mode": "enforce", "policies": "rules: []"}, + ) + + response = service.get_policy(PolicyContext(is_conversational=True)) + + assert response.mode == "enforce" + assert response.policies == "rules: []" + + async def test_get_policy_async_delegates( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/policy", + status_code=200, + json={"mode": "audit", "policies": ""}, + ) + + response = await service.get_policy_async(PolicyContext()) + + assert response.mode == "audit" + + class TestServiceUrlOverride: + """Honor UIPATH_SERVICE_URL_AGENTICGOVERNANCE for local dev.""" + + def test_redirects_policy_fetch_to_override( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv( + "UIPATH_SERVICE_URL_AGENTICGOVERNANCE", "http://localhost:8123" + ) + captured: dict[str, httpx.Request] = {} + + def capture(request: httpx.Request) -> httpx.Response: + captured["request"] = request + return httpx.Response(200, json={"mode": "audit", "policies": ""}) + + httpx_mock.add_callback( + capture, url="http://localhost:8123/api/v1/runtime/policy" + ) + + service.retrieve_policy() + + request = captured["request"] + # Routing headers replace the platform router, org-UUID path is dropped. + assert request.headers["X-UiPath-Internal-TenantId"] == TENANT_ID + assert request.headers["X-UiPath-Internal-AccountId"] == ORG_ID + assert ORG_ID not in str(request.url) + + def test_redirects_compensate_to_override( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv( + "UIPATH_SERVICE_URL_AGENTICGOVERNANCE", "http://localhost:8123" + ) + httpx_mock.add_response( + url="http://localhost:8123/api/v1/runtime/govern", + method="POST", + status_code=200, + json={}, + ) + + service.compensate(**_compensate_kwargs()) + + sent = httpx_mock.get_requests()[-1] + assert sent.method == "POST" + assert sent.headers["X-UiPath-Internal-AccountId"] == ORG_ID + + def test_redirects_track_event_to_override( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv( + "UIPATH_SERVICE_URL_AGENTICGOVERNANCE", "http://localhost:8123" + ) + httpx_mock.add_response( + url="http://localhost:8123/api/v1/runtime/log", + method="POST", + status_code=204, + ) + + service._track_event(event_name="hello", operation_id="op-1") + + sent = httpx_mock.get_requests()[-1] + assert sent.method == "POST" + assert sent.headers["X-UiPath-Internal-AccountId"] == ORG_ID + assert sent.headers["x-uipath-operation-id"] == "op-1" + + class TestTrackEvent: + """Test track_event (sync).""" + + def test_posts_event_name_only_payload( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + ) -> None: + captured: dict[str, httpx.Request] = {} + + def capture(request: httpx.Request) -> httpx.Response: + captured["request"] = request + return httpx.Response(204) + + httpx_mock.add_callback( + capture, + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/log", + ) + + service._track_event(event_name="agent.started") + + request = captured["request"] + assert request.method == "POST" + assert request.headers["x-uipath-internal-tenantid"] == TENANT_ID + assert json.loads(request.content) == {"eventName": "agent.started"} + + def test_includes_data_when_provided( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + ) -> None: + captured: dict[str, httpx.Request] = {} + + def capture(request: httpx.Request) -> httpx.Response: + captured["request"] = request + return httpx.Response(204) + + httpx_mock.add_callback( + capture, + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/log", + ) + + service._track_event( + event_name="agent.completed", + data={"duration_ms": 1234, "outcome": "success"}, + ) + + body = json.loads(captured["request"].content) + assert body == { + "eventName": "agent.completed", + "data": {"duration_ms": 1234, "outcome": "success"}, + } + + def test_sends_caller_operation_id_header( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + ) -> None: + captured: dict[str, httpx.Request] = {} + + def capture(request: httpx.Request) -> httpx.Response: + captured["request"] = request + return httpx.Response(204) + + httpx_mock.add_callback( + capture, + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/log", + ) + + service._track_event(event_name="ev", operation_id="caller-supplied") + + assert captured["request"].headers["x-uipath-operation-id"] == ( + "caller-supplied" + ) + + def test_falls_back_to_resolved_trace_id( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + # When the caller omits operation_id, the header is filled + # from resolve_trace_id() so events join the agent's trace. + monkeypatch.setenv("UIPATH_TRACE_ID", TENANT_ID_HEX) + captured: dict[str, httpx.Request] = {} + + def capture(request: httpx.Request) -> httpx.Response: + captured["request"] = request + return httpx.Response(204) + + httpx_mock.add_callback( + capture, + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/log", + ) + + service._track_event(event_name="ev") + + assert captured["request"].headers["x-uipath-operation-id"] == TENANT_ID_HEX + + def test_caller_operation_id_overrides_trace_fallback( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv("UIPATH_TRACE_ID", TENANT_ID_HEX) + captured: dict[str, httpx.Request] = {} + + def capture(request: httpx.Request) -> httpx.Response: + captured["request"] = request + return httpx.Response(204) + + httpx_mock.add_callback( + capture, + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/log", + ) + + service._track_event(event_name="ev", operation_id="explicit") + + assert captured["request"].headers["x-uipath-operation-id"] == "explicit" + + def test_omits_operation_id_header_when_no_source( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.delenv("UIPATH_TRACE_ID", raising=False) + captured: dict[str, httpx.Request] = {} + + def capture(request: httpx.Request) -> httpx.Response: + captured["request"] = request + return httpx.Response(204) + + httpx_mock.add_callback( + capture, + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/log", + ) + + service._track_event(event_name="ev") + + assert "x-uipath-operation-id" not in captured["request"].headers + + def test_raises_when_org_id_missing( + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.delenv("UIPATH_ORGANIZATION_ID", raising=False) + monkeypatch.setenv("UIPATH_TENANT_ID", TENANT_ID) + service = GovernanceService( + config=config, execution_context=execution_context + ) + + with pytest.raises(ValueError, match="UIPATH_ORGANIZATION_ID"): + service._track_event(event_name="ev") + + def test_raises_on_http_error( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + ) -> None: + from uipath.platform.errors import EnrichedException + + httpx_mock.add_response( + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/log", + status_code=400, + text="bad event", + ) + + with pytest.raises(EnrichedException): + service._track_event(event_name="ev") + + @pytest.mark.parametrize("invalid_name", ["", " ", "\t", "\n"]) + def test_raises_on_empty_event_name( + self, + service: GovernanceService, + invalid_name: str, + ) -> None: + # Fail fast client-side instead of round-tripping a backend + # 400; matches the platform's own non-empty check on + # /runtime/log. + with pytest.raises(ValueError, match="event_name"): + service._track_event(event_name=invalid_name) + + class TestTrackEventAsync: + """Test track_event_async.""" + + async def test_posts_event_and_falls_back_to_trace_id( + self, + httpx_mock: HTTPXMock, + service: GovernanceService, + base_url: str, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv("UIPATH_TRACE_ID", TENANT_ID_HEX) + httpx_mock.add_response( + url=f"{base_url}/{ORG_ID}/agenticgovernance_/api/v1/runtime/log", + status_code=204, + ) + + await service._track_event_async(event_name="ev", data={"foo": "bar"}) + + sent = httpx_mock.get_requests()[-1] + assert sent.method == "POST" + assert sent.headers["x-uipath-operation-id"] == TENANT_ID_HEX + assert json.loads(sent.content) == { + "eventName": "ev", + "data": {"foo": "bar"}, + } + + async def test_raises_on_empty_event_name( + self, service: GovernanceService + ) -> None: + with pytest.raises(ValueError, match="event_name"): + await service._track_event_async(event_name=" ") + + +class TestResolveTraceId: + """Test the resolve_trace_id helper.""" + + def test_returns_fallback_when_no_source_set( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("UIPATH_TRACE_ID", raising=False) + + assert resolve_trace_id(fallback="fallback-id") == "fallback-id" + + def test_returns_none_when_no_fallback( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("UIPATH_TRACE_ID", raising=False) + + assert resolve_trace_id() is None + + def test_reads_uipath_trace_id_in_hex_form( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("UIPATH_TRACE_ID", "0123456789abcdef0123456789abcdef") + + assert resolve_trace_id() == "0123456789abcdef0123456789abcdef" + + def test_normalizes_uipath_trace_id_in_uuid_form( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("UIPATH_TRACE_ID", TENANT_ID) + + assert resolve_trace_id() == TENANT_ID_HEX + + def test_falls_through_when_uipath_trace_id_is_malformed( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("UIPATH_TRACE_ID", "not-a-valid-trace-id") + + # No OTel context active → falls through to caller-supplied fallback. + assert resolve_trace_id(fallback="recovered") == "recovered" diff --git a/packages/uipath-platform/tests/services/test_guardrails_decorators.py b/packages/uipath-platform/tests/services/test_guardrails_decorators.py new file mode 100644 index 000000000..c6cfee2ad --- /dev/null +++ b/packages/uipath-platform/tests/services/test_guardrails_decorators.py @@ -0,0 +1,1319 @@ +"""Tests for the guardrails decorator framework in uipath-platform. + +Focus: meaningful business behaviour — serialization, PRE/POST evaluation, +modification flow, stage enforcement, factory-function path, and integration +scenarios modelled on the joke-agent-decorator sample. +""" + +from __future__ import annotations + +import dataclasses +import logging +from typing import Annotated, Any +from unittest.mock import MagicMock, patch + +import pytest +from pydantic import BaseModel +from uipath.core.guardrails import ( + GuardrailValidationResult, + GuardrailValidationResultType, +) + +from uipath.platform.guardrails.decorators import ( + BlockAction, + ByoValidator, + CustomValidator, + GuardrailAction, + GuardrailBlockException, + GuardrailExclude, + GuardrailExecutionStage, + LogAction, + LoggingSeverityLevel, + PIIDetectionEntity, + PIIDetectionEntityType, + PIIValidator, + PromptInjectionValidator, + guardrail, + register_guardrail_adapter, +) +from uipath.platform.guardrails.decorators._core import ( + _collect_output, + _get_excluded_params, + _make_evaluator, + _reconstruct_output, + _serialize_value, +) +from uipath.platform.guardrails.decorators._registry import ( + _adapters, +) + +# --------------------------------------------------------------------------- +# Shared result constants +# --------------------------------------------------------------------------- + +_PASSED = GuardrailValidationResult( + result=GuardrailValidationResultType.PASSED, + reason="ok", +) +_FAILED = GuardrailValidationResult( + result=GuardrailValidationResultType.VALIDATION_FAILED, + reason="violation detected", +) + + +# --------------------------------------------------------------------------- +# Registry isolation fixture +# --------------------------------------------------------------------------- + + +@pytest.fixture(autouse=True) +def _isolate_adapter_registry(): + """Snapshot and restore the global adapter registry around every test.""" + snapshot = list(_adapters) + yield + _adapters.clear() + _adapters.extend(snapshot) + + +# --------------------------------------------------------------------------- +# Minimal fake types for adapter tests (no LangChain dependency) +# --------------------------------------------------------------------------- + + +class _DummyTarget: + """Minimal callable target recognised by _DummyAdapter.""" + + def __init__(self, return_value: Any = None) -> None: + self.return_value = ( + return_value if return_value is not None else {"output": "result"} + ) + self.invoke_calls: list[Any] = [] + + def invoke(self, args: Any) -> Any: + self.invoke_calls.append(args) + return self.return_value + + +class _WrappedDummyTarget: + """A _DummyTarget wrapped with guardrail evaluation.""" + + def __init__( + self, + target: Any, + evaluator: Any, + action: GuardrailAction, + name: str, + stage: GuardrailExecutionStage, + ) -> None: + self._target = target + self._evaluator = evaluator + self._action = action + self._name = name + self._stage = stage + + def invoke(self, args: Any) -> Any: + input_data = args if isinstance(args, dict) else {"input": args} + if self._stage in ( + GuardrailExecutionStage.PRE, + GuardrailExecutionStage.PRE_AND_POST, + ): + result = self._evaluator( + input_data, GuardrailExecutionStage.PRE, input_data, None + ) + if result.result == GuardrailValidationResultType.VALIDATION_FAILED: + self._action.handle_validation_result(result, input_data, self._name) + raw = self._target.invoke(args) + output_data = raw if isinstance(raw, dict) else {"output": raw} + if self._stage in ( + GuardrailExecutionStage.POST, + GuardrailExecutionStage.PRE_AND_POST, + ): + result = self._evaluator( + output_data, GuardrailExecutionStage.POST, input_data, output_data + ) + if result.result == GuardrailValidationResultType.VALIDATION_FAILED: + self._action.handle_validation_result(result, output_data, self._name) + return raw + + +class _DummyAdapter: + """Adapter that handles _DummyTarget and _WrappedDummyTarget instances.""" + + def recognize(self, target: Any) -> bool: + return isinstance(target, (_DummyTarget, _WrappedDummyTarget)) + + def wrap( + self, + target: Any, + evaluator: Any, + action: GuardrailAction, + name: str, + stage: GuardrailExecutionStage, + ) -> Any: + return _WrappedDummyTarget(target, evaluator, action, name, stage) + + +# --------------------------------------------------------------------------- +# 1. PIIDetectionEntity — threshold boundary enforcement +# --------------------------------------------------------------------------- + + +class TestPIIDetectionEntity: + def test_threshold_below_zero_raises(self): + with pytest.raises(ValueError, match="0.0 and 1.0"): + PIIDetectionEntity(name="Email", threshold=-0.1) + + def test_threshold_above_one_raises(self): + with pytest.raises(ValueError, match="0.0 and 1.0"): + PIIDetectionEntity(name="Email", threshold=1.1) + + +# --------------------------------------------------------------------------- +# 2. LogAction — does NOT stop execution; uses configured severity +# --------------------------------------------------------------------------- + + +class TestLogAction: + def test_violation_logs_guardrail_name_and_execution_continues(self, caplog): + action = LogAction() + with caplog.at_level(logging.WARNING): + result = action.handle_validation_result(_FAILED, "data", "MyGuardrail") + assert result is None # execution continues + assert any("MyGuardrail" in r.message for r in caplog.records) + + def test_pass_emits_no_log(self, caplog): + action = LogAction() + with caplog.at_level(logging.WARNING): + action.handle_validation_result(_PASSED, "data", "G") + assert not caplog.records + + def test_custom_message_overrides_reason(self, caplog): + action = LogAction(message="custom alert") + with caplog.at_level(logging.WARNING): + action.handle_validation_result(_FAILED, "data", "G") + assert any("custom alert" in r.message for r in caplog.records) + + def test_default_message_includes_validation_reason(self, caplog): + action = LogAction() + with caplog.at_level(logging.WARNING): + action.handle_validation_result(_FAILED, "data", "G") + assert any("violation detected" in r.message for r in caplog.records) + + def test_debug_severity(self, caplog): + action = LogAction(severity_level=LoggingSeverityLevel.DEBUG) + with caplog.at_level(logging.DEBUG): + action.handle_validation_result(_FAILED, "data", "G") + debug_records = [r for r in caplog.records if r.levelno == logging.DEBUG] + assert debug_records + + +# --------------------------------------------------------------------------- +# 3. BlockAction — raises GuardrailBlockException on violation +# --------------------------------------------------------------------------- + + +class TestBlockAction: + def test_raises_on_violation(self): + action = BlockAction() + with pytest.raises(GuardrailBlockException): + action.handle_validation_result(_FAILED, "data", "G") + + def test_no_raise_on_pass(self): + action = BlockAction() + result = action.handle_validation_result(_PASSED, "data", "G") + assert result is None + + def test_title_and_detail_from_result(self): + action = BlockAction() + with pytest.raises(GuardrailBlockException) as exc_info: + action.handle_validation_result(_FAILED, "data", "MyGuardrail") + assert exc_info.value.title + assert exc_info.value.detail + + def test_custom_title_and_detail(self): + action = BlockAction(title="Blocked", detail="Not allowed") + with pytest.raises(GuardrailBlockException) as exc_info: + action.handle_validation_result(_FAILED, "data", "G") + assert exc_info.value.title == "Blocked" + assert exc_info.value.detail == "Not allowed" + + +# --------------------------------------------------------------------------- +# 4. PIIValidator — builds correct BuiltInValidatorGuardrail +# --------------------------------------------------------------------------- + + +class TestPIIValidator: + def test_empty_entities_raises(self): + with pytest.raises(ValueError, match="non-empty"): + PIIValidator(entities=[]) + + def test_entity_names_and_thresholds_in_api_parameters(self): + v = PIIValidator( + entities=[ + PIIDetectionEntity(PIIDetectionEntityType.EMAIL, 0.6), + PIIDetectionEntity(PIIDetectionEntityType.PERSON, 0.8), + ] + ) + g = v.get_built_in_guardrail("G", None, True) + param_by_id = {p.id: p for p in g.validator_parameters} + entities_value = param_by_id["entities"].value + assert isinstance(entities_value, list) + assert "Email" in entities_value + assert "Person" in entities_value + thresholds_value = param_by_id["entityThresholds"].value + assert isinstance(thresholds_value, dict) + assert thresholds_value["Email"] == 0.6 + assert thresholds_value["Person"] == 0.8 + + def test_no_scope_restriction(self): + v = PIIValidator(entities=[PIIDetectionEntity(PIIDetectionEntityType.EMAIL)]) + # All stages allowed — no ValueError raised + v.validate_stage(GuardrailExecutionStage.PRE) + v.validate_stage(GuardrailExecutionStage.POST) + + def test_selector_is_none(self): + v = PIIValidator(entities=[PIIDetectionEntity(PIIDetectionEntityType.EMAIL)]) + g = v.get_built_in_guardrail("G", None, True) + assert g.selector is None + + +# --------------------------------------------------------------------------- +# 5. PromptInjectionValidator — LLM-only, PRE-only, threshold validation +# --------------------------------------------------------------------------- + + +class TestPromptInjectionValidator: + def test_threshold_below_zero_raises(self): + with pytest.raises(ValueError, match="threshold"): + PromptInjectionValidator(threshold=-0.1) + + def test_threshold_above_one_raises(self): + with pytest.raises(ValueError, match="threshold"): + PromptInjectionValidator(threshold=1.1) + + def test_restricted_to_pre_stage_only(self): + v = PromptInjectionValidator() + v.validate_stage(GuardrailExecutionStage.PRE) # ok + with pytest.raises(ValueError): + v.validate_stage(GuardrailExecutionStage.POST) + + def test_builds_prompt_injection_guardrail_with_threshold(self): + v = PromptInjectionValidator(threshold=0.7) + g = v.get_built_in_guardrail("PI", None, True) + assert g.validator_type == "prompt_injection" + threshold_param = next(p for p in g.validator_parameters if p.id == "threshold") + assert threshold_param.value == 0.7 + + def test_selector_is_none(self): + v = PromptInjectionValidator() + g = v.get_built_in_guardrail("PI", None, True) + assert g.selector is None + + +# --------------------------------------------------------------------------- +# 6. CustomValidator — rule routing and error handling +# --------------------------------------------------------------------------- + + +class TestCustomValidator: + def test_non_callable_raises(self): + with pytest.raises(ValueError, match="callable"): + CustomValidator(rule="not_a_function") # type: ignore[arg-type] + + def test_wrong_arity_raises(self): + with pytest.raises(ValueError, match="1 or 2"): + CustomValidator(rule=lambda: True) # type: ignore[arg-type] + with pytest.raises(ValueError, match="1 or 2"): + CustomValidator(rule=lambda a, b, c: True) # type: ignore[arg-type] + + def test_one_param_pre_receives_input_data(self): + received: list[Any] = [] + + def capture_pre(args: dict[str, Any]) -> bool: + received.append(args) + return False + + CustomValidator(rule=capture_pre).evaluate( + {}, GuardrailExecutionStage.PRE, {"a": 1}, None + ) + assert received == [{"a": 1}] + + def test_one_param_post_receives_output_data(self): + received: list[Any] = [] + + def capture_post(args: dict[str, Any]) -> bool: + received.append(args) + return False + + CustomValidator(rule=capture_post).evaluate( + {}, GuardrailExecutionStage.POST, {"in": 1}, {"out": 2} + ) + assert received == [{"out": 2}] + + def test_two_param_post_receives_input_and_output(self): + received: list[Any] = [] + + def rule(inp: dict[str, Any], out: dict[str, Any]) -> bool: + received.append((inp, out)) + return False + + CustomValidator(rule=rule).evaluate( + {}, GuardrailExecutionStage.POST, {"in": 1}, {"out": 2} + ) + assert received == [({"in": 1}, {"out": 2})] + + def test_two_param_rule_skipped_when_input_missing(self): + result = CustomValidator(rule=lambda a, b: True).evaluate( + {}, GuardrailExecutionStage.POST, None, {"out": 2} + ) + assert result.result == GuardrailValidationResultType.PASSED + + def test_rule_returning_true_means_violation(self): + result = CustomValidator(rule=lambda args: True).evaluate( + {}, GuardrailExecutionStage.PRE, {"x": 1}, None + ) + assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + + def test_rule_exception_returns_passed(self): + def bad(args: dict[str, Any]) -> bool: + raise ValueError("boom") + + result = CustomValidator(rule=bad).evaluate( + {}, GuardrailExecutionStage.PRE, {"x": 1}, None + ) + assert result.result == GuardrailValidationResultType.PASSED + + +# --------------------------------------------------------------------------- +# 7. GuardrailExclude — parameter introspection +# --------------------------------------------------------------------------- + + +class TestGuardrailExclude: + def test_excluded_param_not_in_collected_input(self): + def func( + text: str, + config: Annotated[dict[str, Any], GuardrailExclude()], + ) -> str: + return text + + excluded = _get_excluded_params(func) + assert "config" in excluded + assert "text" not in excluded + + def test_multiple_excluded_params(self): + def func( + a: str, + b: Annotated[int, GuardrailExclude()], + c: Annotated[str, GuardrailExclude()], + ) -> str: + return a + + excluded = _get_excluded_params(func) + assert excluded == {"b", "c"} + + def test_no_annotations_returns_empty_set(self): + def func(a: str, b: int) -> str: + return a + + assert _get_excluded_params(func) == set() + + +# --------------------------------------------------------------------------- +# 8. Serialization helpers +# --------------------------------------------------------------------------- + + +class _PydanticModel(BaseModel): + topic: str + count: int = 0 + + +@dataclasses.dataclass +class _Dataclass: + name: str + value: float + + +class TestSerializationHelpers: + def test_primitive_str_passthrough(self): + assert _serialize_value("hello") == "hello" + + def test_primitive_int_passthrough(self): + assert _serialize_value(42) == 42 + + def test_dict_passthrough(self): + assert _serialize_value({"a": 1}) == {"a": 1} + + def test_pydantic_model_dumps(self): + m = _PydanticModel(topic="test", count=3) + result = _serialize_value(m) + assert result == {"topic": "test", "count": 3} + + def test_dataclass_asdict(self): + d = _Dataclass(name="x", value=1.5) + result = _serialize_value(d) + assert result == {"name": "x", "value": 1.5} + + def test_collect_output_from_pydantic(self): + m = _PydanticModel(topic="joke") + result = _collect_output(m) + assert result == {"topic": "joke", "count": 0} + + def test_collect_output_from_str(self): + result = _collect_output("hello") + assert result == {"return": "hello"} + + def test_collect_output_from_dict(self): + result = _collect_output({"key": "val"}) + assert result == {"key": "val"} + + def test_reconstruct_output_pydantic(self): + original = _PydanticModel(topic="original") + modified = {"topic": "modified", "count": 5} + result = _reconstruct_output(original, modified) + assert isinstance(result, _PydanticModel) + assert result.topic == "modified" + assert result.count == 5 + + def test_reconstruct_output_str(self): + result = _reconstruct_output("original", "modified") + assert result == "modified" + + def test_reconstruct_output_none_returns_original(self): + result = _reconstruct_output("original", None) + assert result == "original" + + +# --------------------------------------------------------------------------- +# 9. @guardrail on sync functions +# --------------------------------------------------------------------------- + + +class TestGuardrailOnSyncFunction: + def test_pre_fires_before_function(self): + calls: list[str] = [] + + mock_validator = MagicMock() + mock_validator.supported_stages = [] + mock_validator.validate_stage = MagicMock() + + mock_validator.run.side_effect = lambda *a, **kw: ( + calls.append("eval") or _PASSED # type: ignore[func-returns-value] + ) + + @guardrail( + validator=mock_validator, + action=LogAction(), + stage=GuardrailExecutionStage.PRE, + ) + def fn(text: str) -> str: + calls.append("fn") + return text + + fn("hello") + assert calls == ["eval", "fn"] + + def test_post_fires_after_function(self): + calls: list[str] = [] + + mock_validator = MagicMock() + mock_validator.supported_stages = [] + mock_validator.validate_stage = MagicMock() + + mock_validator.run.side_effect = lambda *a, **kw: ( + calls.append("eval") or _PASSED # type: ignore[func-returns-value] + ) + + @guardrail( + validator=mock_validator, + action=LogAction(), + stage=GuardrailExecutionStage.POST, + ) + def fn(text: str) -> str: + calls.append("fn") + return text + + fn("hello") + assert calls == ["fn", "eval"] + + def test_block_action_raises_guardrail_block_exception(self): + mock_validator = MagicMock() + mock_validator.supported_stages = [] + mock_validator.validate_stage = MagicMock() + + mock_validator.run.return_value = _FAILED + + @guardrail( + validator=mock_validator, + action=BlockAction(title="Blocked", detail="not allowed"), + stage=GuardrailExecutionStage.PRE, + ) + def fn(text: str) -> str: + return text + + with pytest.raises(GuardrailBlockException): + fn("bad input") + + def test_log_action_does_not_stop_execution(self): + mock_validator = MagicMock() + mock_validator.supported_stages = [] + mock_validator.validate_stage = MagicMock() + + mock_validator.run.return_value = _FAILED + + result = [] + + @guardrail( + validator=mock_validator, + action=LogAction(), + stage=GuardrailExecutionStage.PRE, + ) + def fn(text: str) -> str: + result.append("called") + return text + + fn("input") + assert result == ["called"] + + def test_pre_input_contains_function_params(self): + captured: list[Any] = [] + + @guardrail( + validator=CustomValidator( + lambda args: captured.append(args) or False # type: ignore[func-returns-value] + ), + action=LogAction(), + stage=GuardrailExecutionStage.PRE, + ) + def fn(joke: str, count: int) -> str: + return joke + + fn("why did the chicken", 3) + assert captured == [{"joke": "why did the chicken", "count": 3}] + + def test_excluded_param_absent_from_pre_input(self): + captured: list[Any] = [] + + @guardrail( + validator=CustomValidator( + lambda args: captured.append(args) or False # type: ignore[func-returns-value] + ), + action=LogAction(), + stage=GuardrailExecutionStage.PRE, + ) + def fn( + joke: str, + config: Annotated[dict[str, Any], GuardrailExclude()], + ) -> str: + return joke + + fn("why did the chicken", {"debug": True}) + assert "config" not in captured[0] + assert "joke" in captured[0] + + def test_pre_modification_updates_function_args(self): + class _ReplaceAction(GuardrailAction): + def handle_validation_result(self, result, data, name): + if isinstance(data, dict) and "joke" in data: + return {"joke": data["joke"].replace("donkey", "[censored]")} + return data + + @guardrail( + validator=CustomValidator(lambda args: "donkey" in args.get("joke", "")), + action=_ReplaceAction(), + stage=GuardrailExecutionStage.PRE, + ) + def fn(joke: str) -> str: + return joke + + result = fn("why did the donkey cross the road") + assert result == "why did the [censored] cross the road" + + def test_post_output_contains_return_value(self): + captured: list[Any] = [] + + @guardrail( + validator=CustomValidator( + lambda args: captured.append(args) or False # type: ignore[func-returns-value] + ), + action=LogAction(), + stage=GuardrailExecutionStage.POST, + ) + def fn(x: int) -> dict[str, int]: + return {"result": x * 2} + + fn(5) + assert captured == [{"result": 10}] + + def test_post_modification_updates_return_value(self): + class _FixedAction(GuardrailAction): + def handle_validation_result(self, result, data, name): + return {"result": 99} + + @guardrail( + validator=CustomValidator(lambda args: True), + action=_FixedAction(), + stage=GuardrailExecutionStage.POST, + ) + def fn(x: int) -> dict[str, int]: + return {"result": x * 2} + + assert fn(5) == {"result": 99} + + def test_pre_and_post_both_fire(self): + calls: list[str] = [] + + mock_validator = MagicMock() + mock_validator.supported_stages = [] + mock_validator.validate_stage = MagicMock() + + mock_validator.run.side_effect = lambda name, desc, enabled, data, stage, *a: ( + calls.append(stage.value) or _PASSED # type: ignore[func-returns-value] + ) + + @guardrail( + validator=mock_validator, + action=LogAction(), + stage=GuardrailExecutionStage.PRE_AND_POST, + ) + def fn(x: int) -> int: + return x + 1 + + fn(1) + assert "pre" in calls + assert "post" in calls + + +# --------------------------------------------------------------------------- +# 10. @guardrail on async functions +# --------------------------------------------------------------------------- + + +class TestGuardrailOnAsyncFunction: + @pytest.mark.asyncio + async def test_pre_fires_before_async_function(self): + calls: list[str] = [] + + mock_validator = MagicMock() + mock_validator.supported_stages = [] + mock_validator.validate_stage = MagicMock() + + mock_validator.run.side_effect = lambda *a, **kw: ( + calls.append("eval") or _PASSED # type: ignore[func-returns-value] + ) + + @guardrail( + validator=mock_validator, + action=LogAction(), + stage=GuardrailExecutionStage.PRE, + ) + async def fn(text: str) -> str: + calls.append("fn") + return text + + await fn("hello") + assert calls == ["eval", "fn"] + + @pytest.mark.asyncio + async def test_block_action_raises_in_async(self): + mock_validator = MagicMock() + mock_validator.supported_stages = [] + mock_validator.validate_stage = MagicMock() + + mock_validator.run.return_value = _FAILED + + @guardrail( + validator=mock_validator, + action=BlockAction(), + stage=GuardrailExecutionStage.PRE, + ) + async def fn(text: str) -> str: + return text + + with pytest.raises(GuardrailBlockException): + await fn("bad") + + @pytest.mark.asyncio + async def test_post_modification_in_async(self): + class _FortyTwoAction(GuardrailAction): + def handle_validation_result(self, result, data, name): + return {"result": 42} + + @guardrail( + validator=CustomValidator(lambda args: True), + action=_FortyTwoAction(), + stage=GuardrailExecutionStage.POST, + ) + async def fn(x: int) -> dict[str, int]: + return {"result": x} + + assert await fn(1) == {"result": 42} + + +# --------------------------------------------------------------------------- +# 11. Stage enforcement at decoration time +# --------------------------------------------------------------------------- + + +class TestStageEnforcement: + def test_prompt_injection_on_post_raises_at_decoration(self): + with pytest.raises(ValueError, match="stage"): + guardrail( + lambda text: text, + validator=PromptInjectionValidator(), + action=LogAction(), + stage=GuardrailExecutionStage.POST, + ) + + def test_prompt_injection_on_pre_and_post_raises_at_decoration(self): + with pytest.raises(ValueError, match="stage"): + guardrail( + lambda text: text, + validator=PromptInjectionValidator(), + action=LogAction(), + stage=GuardrailExecutionStage.PRE_AND_POST, + ) + + def test_prompt_injection_on_pre_ok(self): + # Should not raise + guardrail( + lambda text: text, + validator=PromptInjectionValidator(), + action=LogAction(), + stage=GuardrailExecutionStage.PRE, + ) + + +# --------------------------------------------------------------------------- +# 12. @guardrail validation (bad arguments) +# --------------------------------------------------------------------------- + + +class TestGuardrailDecorator: + def test_missing_action_raises(self): + with pytest.raises(ValueError, match="action must be provided"): + guardrail( + lambda text: text, + validator=CustomValidator(lambda args: False), + action=None, # type: ignore[arg-type] + ) + + def test_non_action_instance_raises(self): + with pytest.raises(ValueError, match="GuardrailAction"): + guardrail( + lambda text: text, + validator=CustomValidator(lambda args: False), + action="bad", # type: ignore[arg-type] + ) + + def test_invalid_enabled_for_evals_type_raises(self): + with pytest.raises(ValueError, match="boolean"): + guardrail( + lambda text: text, + validator=CustomValidator(lambda args: False), + action=LogAction(), + enabled_for_evals="yes", # type: ignore[arg-type] + ) + + +# --------------------------------------------------------------------------- +# 13. Stacked decorators +# --------------------------------------------------------------------------- + + +class TestStackedDecorators: + def test_both_decorators_fire_on_same_function(self): + calls: list[str] = [] + + def _make_mock(tag: str) -> Any: + m = MagicMock() + m.supported_stages = [] + m.validate_stage = MagicMock() + m.get_built_in_guardrail.return_value = None + m.run.side_effect = lambda *a, **kw: calls.append(tag) or _PASSED # type: ignore[func-returns-value] + return m + + @guardrail( + validator=_make_mock("outer"), + action=LogAction(), + stage=GuardrailExecutionStage.PRE, + name="outer", + ) + @guardrail( + validator=_make_mock("inner"), + action=LogAction(), + stage=GuardrailExecutionStage.PRE, + name="inner", + ) + def fn(text: str) -> str: + return text + + fn("hello") + assert "outer" in calls + assert "inner" in calls + + def test_outer_block_prevents_inner_from_firing(self): + inner_called = [] + + outer_validator = MagicMock() + outer_validator.supported_stages = [] + outer_validator.validate_stage = MagicMock() + outer_validator.get_built_in_guardrail.return_value = None + outer_validator.run.return_value = _FAILED + + inner_validator = MagicMock() + inner_validator.supported_stages = [] + inner_validator.validate_stage = MagicMock() + inner_validator.get_built_in_guardrail.return_value = None + inner_validator.run.side_effect = lambda *a, **kw: ( + inner_called.append(True) or _PASSED # type: ignore[func-returns-value] + ) + + @guardrail( + validator=outer_validator, + action=BlockAction(), + stage=GuardrailExecutionStage.PRE, + ) + @guardrail( + validator=inner_validator, + action=LogAction(), + stage=GuardrailExecutionStage.PRE, + ) + def fn(text: str) -> str: + return text + + with pytest.raises(GuardrailBlockException): + fn("bad") + + assert not inner_called + + +# --------------------------------------------------------------------------- +# 14. Factory function path (adapter wraps return value) +# --------------------------------------------------------------------------- + + +class TestFactoryFunctionPath: + def test_adapter_wraps_return_value_of_factory(self): + register_guardrail_adapter(_DummyAdapter()) + target = _DummyTarget(return_value={"output": "ok"}) + + @guardrail( + validator=CustomValidator(lambda args: False), + action=LogAction(), + stage=GuardrailExecutionStage.PRE, + ) + def factory() -> _DummyTarget: + return target + + wrapped = factory() + assert isinstance(wrapped, _WrappedDummyTarget) + + def test_factory_pre_guardrail_fires_on_factory_params(self): + register_guardrail_adapter(_DummyAdapter()) + captured: list[Any] = [] + target = _DummyTarget() + + @guardrail( + validator=CustomValidator( + lambda args: captured.append(args) or False # type: ignore[func-returns-value] + ), + action=LogAction(), + stage=GuardrailExecutionStage.PRE, + ) + def factory(config: str) -> _DummyTarget: + return target + + factory("test-config") + assert captured == [{"config": "test-config"}] + + def test_adapter_recognizes_direct_object(self): + register_guardrail_adapter(_DummyAdapter()) + target = _DummyTarget() + + wrapped = guardrail( + target, + validator=CustomValidator(lambda args: False), + action=LogAction(), + stage=GuardrailExecutionStage.PRE, + ) + assert isinstance(wrapped, _WrappedDummyTarget) + + def test_stacked_guardrails_on_factory_both_wrap_return_value(self): + register_guardrail_adapter(_DummyAdapter()) + target = _DummyTarget() + evals: list[str] = [] + + def _make_mock(tag: str) -> Any: + m = MagicMock() + m.supported_stages = [] + m.validate_stage = MagicMock() + m.get_built_in_guardrail.return_value = None + m.run.side_effect = lambda *a, **kw: evals.append(tag) or _PASSED # type: ignore[func-returns-value] + return m + + @guardrail( + validator=_make_mock("outer"), + action=LogAction(), + stage=GuardrailExecutionStage.PRE, + ) + @guardrail( + validator=_make_mock("inner"), + action=LogAction(), + stage=GuardrailExecutionStage.PRE, + ) + def factory() -> _DummyTarget: + return target + + wrapped = factory() + wrapped.invoke({"x": 1}) + assert "outer" in evals + assert "inner" in evals + + +# --------------------------------------------------------------------------- +# 15. _make_evaluator — local vs API path +# --------------------------------------------------------------------------- + + +class TestMakeEvaluator: + def test_custom_validator_path_delegates_to_run(self): + """_make_evaluator with a CustomGuardrailValidator calls validator.run().""" + mock_validator = MagicMock() + mock_validator.run.return_value = _PASSED + evaluator = _make_evaluator(mock_validator, "G", None, True) + result = evaluator({"data": 1}, GuardrailExecutionStage.PRE, {"a": 1}, None) + mock_validator.run.assert_called_once_with( + "G", None, True, {"data": 1}, GuardrailExecutionStage.PRE, {"a": 1}, None + ) + assert result == _PASSED + + def test_built_in_validator_path_lazy_initializes_uipath(self): + """BuiltInGuardrailValidator.run() lazily creates UiPath() and calls API.""" + from uipath.platform.guardrails.decorators.validators import ( + BuiltInGuardrailValidator, + ) + from uipath.platform.guardrails.guardrails import BuiltInValidatorGuardrail + + mock_built_in = MagicMock(spec=BuiltInValidatorGuardrail) + + class _TestBuiltIn(BuiltInGuardrailValidator): + def get_built_in_guardrail(self, name, description, enabled_for_evals): + return mock_built_in + + validator = _TestBuiltIn() + evaluator = _make_evaluator(validator, "G", None, True) + + mock_uipath = MagicMock() + mock_uipath.guardrails.evaluate_guardrail.return_value = _PASSED + with patch("uipath.platform.UiPath", return_value=mock_uipath): + evaluator({"text": "hello"}, GuardrailExecutionStage.PRE, None, None) + evaluator({"text": "hello"}, GuardrailExecutionStage.PRE, None, None) + + # UiPath() should be created only once despite two calls + assert mock_uipath.guardrails.evaluate_guardrail.call_count == 2 + + +# --------------------------------------------------------------------------- +# 16. Joke-agent integration scenarios (plain functions) +# --------------------------------------------------------------------------- + + +class _JokeInput(BaseModel): + topic: str + + +class _JokeOutput(BaseModel): + joke: str + + +class TestJokeAgentScenarios: + """Integration tests modelled on the joke-agent-decorator sample.""" + + def test_pii_validator_blocks_person_name_in_topic(self): + """Agent-level PRE guardrail blocks person names in the input topic.""" + calls: list[str] = [] + + mock_validator = MagicMock() + mock_validator.supported_stages = [] + mock_validator.validate_stage = MagicMock() + + mock_validator.run.return_value = _FAILED + + @guardrail( + validator=mock_validator, + action=BlockAction(title="Person detected", detail="Not allowed"), + stage=GuardrailExecutionStage.PRE, + name="Agent PII", + ) + async def joke_node(state: _JokeInput) -> _JokeOutput: + calls.append("called") + return _JokeOutput(joke="a joke") + + import asyncio + + with pytest.raises(GuardrailBlockException): + asyncio.run(joke_node(_JokeInput(topic="John Smith"))) + assert not calls + + def test_input_pydantic_model_serialized_for_guardrail(self): + """State Pydantic model is serialized to dict and sent to evaluator.""" + captured: list[Any] = [] + + @guardrail( + validator=CustomValidator( + lambda args: captured.append(args) or False # type: ignore[func-returns-value] + ), + action=LogAction(), + stage=GuardrailExecutionStage.PRE, + ) + def process(state: _JokeInput) -> _JokeOutput: + return _JokeOutput(joke="a joke") + + process(_JokeInput(topic="cats")) + assert captured == [{"state": {"topic": "cats"}}] + + def test_output_pydantic_model_serialized_for_guardrail(self): + """Return Pydantic model is serialized to dict and sent to POST evaluator.""" + captured: list[Any] = [] + + @guardrail( + validator=CustomValidator( + lambda args: captured.append(args) or False # type: ignore[func-returns-value] + ), + action=LogAction(), + stage=GuardrailExecutionStage.POST, + ) + def process(state: _JokeInput) -> _JokeOutput: + return _JokeOutput(joke="funny joke about cats") + + process(_JokeInput(topic="cats")) + assert captured == [{"joke": "funny joke about cats"}] + + def test_excluded_config_param_not_in_guardrail_input(self): + """RunnableConfig-style param excluded from evaluation.""" + captured: list[Any] = [] + + @guardrail( + validator=CustomValidator( + lambda args: captured.append(args) or False # type: ignore[func-returns-value] + ), + action=LogAction(), + stage=GuardrailExecutionStage.PRE, + ) + def process( + state: _JokeInput, + config: Annotated[dict[str, Any], GuardrailExclude()], + ) -> _JokeOutput: + return _JokeOutput(joke="joke") + + process(_JokeInput(topic="dogs"), {"thread_id": "abc"}) + assert "config" not in captured[0] + assert "state" in captured[0] + + def test_word_filter_custom_validator_on_tool_function(self): + """CustomValidator on plain function replaces offensive word via action.""" + censored: list[str] = [] + + class CensorAction(GuardrailAction): + def handle_validation_result(self, result, data, name): + if isinstance(data, dict) and "joke" in data: + censored.append(data["joke"]) + return {"joke": data["joke"].replace("donkey", "[censored]")} + return data + + @guardrail( + validator=CustomValidator( + lambda args: "donkey" in args.get("joke", "").lower() + ), + action=CensorAction(), + stage=GuardrailExecutionStage.PRE, + name="Word Filter", + ) + def analyze_joke(joke: str) -> str: + return f"analyzed: {joke}" + + result = analyze_joke(joke="why did the donkey cross the road") + assert "censored" in result + assert "donkey" not in result + + def test_log_action_does_not_stop_joke_generation(self): + """LogAction on PII violation logs but lets execution continue.""" + + @guardrail( + validator=CustomValidator(lambda args: True), # always violate + action=LogAction(), + stage=GuardrailExecutionStage.PRE, + name="Always-Log", + ) + def generate_joke(topic: str) -> str: + return f"joke about {topic}" + + result = generate_joke("cats") + assert result == "joke about cats" + + def test_length_limiter_blocks_long_joke(self): + """BlockAction on length check raises for over-long content.""" + + @guardrail( + validator=CustomValidator(lambda args: len(args.get("joke", "")) > 10), + action=BlockAction(title="Too long", detail="Joke exceeds limit"), + stage=GuardrailExecutionStage.PRE, + ) + def submit_joke(joke: str) -> str: + return joke + + with pytest.raises(GuardrailBlockException, match="Too long"): + submit_joke(joke="a" * 20) + + # Short joke passes through + assert submit_joke(joke="short") == "short" + + +# --------------------------------------------------------------------------- +# LLMAsJudgeValidator via @guardrail +# --------------------------------------------------------------------------- + + +class TestLLMAsJudgeDecorator: + """@guardrail(validator=LLMAsJudgeValidator(...)) behavior on a mocked verdict.""" + + def test_block_action_raises_on_failed_verdict(self): + mock_validator = MagicMock() + mock_validator.supported_stages = [] + mock_validator.validate_stage = MagicMock() + mock_validator.run.return_value = _FAILED + + @guardrail( + validator=mock_validator, + action=BlockAction(title="Blocked", detail="off-topic"), + stage=GuardrailExecutionStage.POST, + ) + def joke(topic: str) -> str: + return f"joke about {topic}" + + with pytest.raises(GuardrailBlockException, match="Blocked"): + joke("cats") + + def test_log_action_continues_on_failed_verdict(self, caplog): + mock_validator = MagicMock() + mock_validator.supported_stages = [] + mock_validator.validate_stage = MagicMock() + mock_validator.run.return_value = _FAILED + + @guardrail( + validator=mock_validator, + action=LogAction(), + stage=GuardrailExecutionStage.PRE, + name="Judge", + ) + def joke(topic: str) -> str: + return f"joke about {topic}" + + with caplog.at_level(logging.WARNING): + assert joke("cats") == "joke about cats" # log does not block + assert any("Judge" in r.message for r in caplog.records) + + def test_real_validator_block_end_to_end_mocked_backend(self): + """Real LLMAsJudgeValidator.run() path with the UiPath API mocked.""" + from uipath.platform.guardrails.decorators import LLMAsJudgeValidator + + mock_uipath = MagicMock() + mock_uipath.guardrails.evaluate_guardrail.return_value = _FAILED + + @guardrail( + validator=LLMAsJudgeValidator( + guardrail_text="Must be on-topic.", model="gpt-4o-2024-08-06" + ), + action=BlockAction(title="Off-topic", detail="blocked"), + stage=GuardrailExecutionStage.POST, + ) + def joke(topic: str) -> str: + return f"joke about {topic}" + + with patch("uipath.platform.UiPath", return_value=mock_uipath): + with pytest.raises(GuardrailBlockException): + joke("cats") + mock_uipath.guardrails.evaluate_guardrail.assert_called_once() + + +# --------------------------------------------------------------------------- +# ByoValidator — Bring Your Own Guardrail configuration reference +# --------------------------------------------------------------------------- + + +class TestByoValidator: + def test_empty_validator_name_raises(self): + with pytest.raises(ValueError, match="validator_name"): + ByoValidator("") + + def test_whitespace_validator_name_raises(self): + with pytest.raises(ValueError, match="validator_name"): + ByoValidator(" ") + + def test_builds_byo_guardrail_from_name_alone(self): + v = ByoValidator("my-harmful-content-guardrail") + g = v.get_built_in_guardrail("G", None, True) + assert g.validator_type == "byo" + assert g.byo_validator_name == "my-harmful-content-guardrail" + + def test_aliases_serialize_for_the_wire(self): + v = ByoValidator("byog-pii") + g = v.get_built_in_guardrail("G", None, True) + dumped = g.model_dump(by_alias=True) + assert dumped["validatorType"] == "byo" + assert dumped["byoValidatorName"] == "byog-pii" + assert dumped["$guardrailType"] == "builtInValidator" + # BYOG resolves by validator name alone (unique per tenant); no + # connection id exists on the wire model. + assert "byoConnectionId" not in dumped + + def test_parameters_pass_through(self): + from uipath.platform.guardrails.guardrails import NumberParameterValue + + param = NumberParameterValue(parameter_type="number", id="threshold", value=0.7) + v = ByoValidator("byog-custom", parameters=[param]) + g = v.get_built_in_guardrail("G", None, True) + assert g.validator_parameters == [param] + + def test_parameters_default_empty(self): + v = ByoValidator("byog-custom") + g = v.get_built_in_guardrail("G", None, True) + assert g.validator_parameters == [] + + def test_default_description_includes_validator_name(self): + v = ByoValidator("byog-harmful-content") + g = v.get_built_in_guardrail("G", None, True) + assert g.description is not None + assert "byog-harmful-content" in g.description + + def test_no_stage_restriction(self): + v = ByoValidator("byog-harmful-content") + # BYO capabilities are connector-defined — all stages allowed + v.validate_stage(GuardrailExecutionStage.PRE) + v.validate_stage(GuardrailExecutionStage.POST) + + def test_selector_is_none(self): + v = ByoValidator("byog-harmful-content") + g = v.get_built_in_guardrail("G", None, True) + assert g.selector is None + + def test_run_forwards_byo_guardrail_to_service(self): + v = ByoValidator("my-harmful-content-guardrail") + mock_uipath = MagicMock() + mock_uipath.guardrails.evaluate_guardrail.return_value = ( + GuardrailValidationResult( + result=GuardrailValidationResultType.PASSED, reason="" + ) + ) + with patch("uipath.platform.UiPath", return_value=mock_uipath): + result = v.run( + "G", None, True, "some input", GuardrailExecutionStage.PRE, None, None + ) + assert result.result == GuardrailValidationResultType.PASSED + data, g = mock_uipath.guardrails.evaluate_guardrail.call_args[0] + assert data == "some input" + assert g.validator_type == "byo" + assert g.byo_validator_name == "my-harmful-content-guardrail" diff --git a/packages/uipath-platform/tests/services/test_guardrails_service.py b/packages/uipath-platform/tests/services/test_guardrails_service.py index 9d8f5a900..d20d531a7 100644 --- a/packages/uipath-platform/tests/services/test_guardrails_service.py +++ b/packages/uipath-platform/tests/services/test_guardrails_service.py @@ -10,6 +10,7 @@ ) from uipath.platform import UiPathApiConfig, UiPathExecutionContext +from uipath.platform.common import ExecutionSourceContext from uipath.platform.guardrails import ( BuiltInValidatorGuardrail, EnumListParameterValue, @@ -262,15 +263,18 @@ def capture_request(request): # Parse the request payload request_payload = json.loads(captured_request.content) - # Verify the payload structure matches the reverted format: + # Verify the payload structure: # { # "validator": guardrail.validator_type, # "input": input_data, # "parameters": parameters, + # "guardrailName": guardrail.name, # } assert "validator" in request_payload assert "input" in request_payload assert "parameters" in request_payload + assert "guardrailName" in request_payload + assert request_payload["guardrailName"] == "PII detection guardrail" # Verify validator is a string (not an object) assert isinstance(request_payload["validator"], str) @@ -298,3 +302,598 @@ def capture_request(request): # Verify result fields assert result.result == GuardrailValidationResultType.PASSED assert result.reason == "Validation passed" + + def test_evaluate_guardrail_byog_forwards_byo_validator_name( + self, + httpx_mock: HTTPXMock, + service: GuardrailsService, + base_url: str, + org: str, + tenant: str, + ) -> None: + """A BYOG guardrail forwards byoValidatorName so the guardrails service + can resolve the connector-backed configuration.""" + captured_request = None + + def capture_request(request): + nonlocal captured_request + captured_request = request + return httpx.Response( + status_code=200, + json={"result": "PASSED", "details": "Validation passed"}, + ) + + httpx_mock.add_callback( + method="POST", + url=f"{base_url}{org}{tenant}/agentsruntime_/api/execution/guardrails/validate", + callback=capture_request, + ) + + # BYOG persisted shape: "byo" sentinel + byoValidatorName reference. + byog_guardrail = BuiltInValidatorGuardrail( + id="byog-id", + name="Databricks PII (BYOG)", + description="Customer-provided PII validator", + enabled_for_evals=True, + selector=GuardrailSelector(scopes=[GuardrailScope.LLM]), + guardrail_type="builtInValidator", + validator_type="byo", + byo_validator_name="my_databricks_pii", + validator_parameters=[], + ) + + result = service.evaluate_guardrail("some input", byog_guardrail) + + assert captured_request is not None + request_payload = json.loads(captured_request.content) + assert request_payload["validator"] == "byo" + assert request_payload["byoValidatorName"] == "my_databricks_pii" + assert result.result == GuardrailValidationResultType.PASSED + + def test_evaluate_guardrail_byog_never_forwards_legacy_connection_id( + self, + httpx_mock: HTTPXMock, + service: GuardrailsService, + base_url: str, + org: str, + tenant: str, + ) -> None: + """A legacy byoConnectionId (from persisted json or an older caller) + is never forwarded: BYOG resolves by validator name alone, which is + unique per tenant; the connection comes from the configuration + server-side.""" + captured_request = None + + def capture_request(request): + nonlocal captured_request + captured_request = request + return httpx.Response( + status_code=200, + json={"result": "PASSED", "details": "Validation passed"}, + ) + + httpx_mock.add_callback( + method="POST", + url=f"{base_url}{org}{tenant}/agentsruntime_/api/execution/guardrails/validate", + callback=capture_request, + ) + + byog_guardrail = BuiltInValidatorGuardrail.model_validate( + { + "$guardrailType": "builtInValidator", + "id": "byog-id", + "name": "Databricks PII (BYOG)", + "enabledForEvals": True, + "selector": {"scopes": ["Llm"]}, + "validatorType": "byo", + "byoValidatorName": "my_databricks_pii", + "byoConnectionId": "byog-conn-1", + "validatorParameters": [], + } + ) + + service.evaluate_guardrail("some input", byog_guardrail) + + assert captured_request is not None + request_payload = json.loads(captured_request.content) + assert request_payload["byoValidatorName"] == "my_databricks_pii" + assert "byoConnectionId" not in request_payload + + def test_evaluate_guardrail_byog_resolves_by_name_alone( + self, + httpx_mock: HTTPXMock, + service: GuardrailsService, + base_url: str, + org: str, + tenant: str, + ) -> None: + """A BYOG guardrail carries only byoValidatorName; no connection id + appears in the payload.""" + captured_request = None + + def capture_request(request): + nonlocal captured_request + captured_request = request + return httpx.Response( + status_code=200, + json={"result": "PASSED", "details": "Validation passed"}, + ) + + httpx_mock.add_callback( + method="POST", + url=f"{base_url}{org}{tenant}/agentsruntime_/api/execution/guardrails/validate", + callback=capture_request, + ) + + byog_guardrail = BuiltInValidatorGuardrail( + id="byog-id", + name="Databricks PII (BYOG)", + enabled_for_evals=True, + selector=GuardrailSelector(scopes=[GuardrailScope.LLM]), + guardrail_type="builtInValidator", + validator_type="byo", + byo_validator_name="my_databricks_pii", + validator_parameters=[], + ) + + service.evaluate_guardrail("some input", byog_guardrail) + + assert captured_request is not None + request_payload = json.loads(captured_request.content) + assert "byoConnectionId" not in request_payload + + def test_evaluate_guardrail_legacy_byo_connection_id_still_parses( + self, + ) -> None: + """Legacy json carrying byoConnectionId still parses (extra="allow"), + but the value is not a typed field anymore.""" + guardrail = BuiltInValidatorGuardrail.model_validate( + { + "$guardrailType": "builtInValidator", + "id": "byog-id", + "name": "BYOG", + "validatorType": "byo", + "byoValidatorName": "my_databricks_pii", + "byoConnectionId": "byog-conn-1", + "validatorParameters": [], + } + ) + assert guardrail.byo_validator_name == "my_databricks_pii" + assert "byo_connection_id" not in type(guardrail).model_fields + + def test_evaluate_guardrail_byo_without_name_raises( + self, + service: GuardrailsService, + ) -> None: + """A "byo" guardrail missing its byoValidatorName reference fails fast + rather than sending an unresolvable request.""" + guardrail = BuiltInValidatorGuardrail( + id="byog-id", + name="Broken BYOG", + enabled_for_evals=True, + selector=GuardrailSelector(scopes=[GuardrailScope.LLM]), + guardrail_type="builtInValidator", + validator_type="byo", + validator_parameters=[], + ) + + with pytest.raises(ValueError, match="byo_validator_name"): + service.evaluate_guardrail("some input", guardrail) + + def test_evaluate_guardrail_byo_validator_name_from_alias(self) -> None: + """byoValidatorName parses into the typed field via its camelCase alias.""" + guardrail = BuiltInValidatorGuardrail.model_validate( + { + "$guardrailType": "builtInValidator", + "id": "byog-id", + "name": "BYOG", + "validatorType": "byo", + "byoValidatorName": "my_databricks_pii", + "validatorParameters": [], + } + ) + assert guardrail.byo_validator_name == "my_databricks_pii" + + def test_evaluate_guardrail_non_byo_type_does_not_forward_name( + self, + httpx_mock: HTTPXMock, + service: GuardrailsService, + base_url: str, + org: str, + tenant: str, + ) -> None: + """byoValidatorName is only forwarded for the "byo" sentinel, never leaked + into a non-BYOG validator payload even if the field happens to be set.""" + captured_request = None + + def capture_request(request): + nonlocal captured_request + captured_request = request + return httpx.Response( + status_code=200, + json={"result": "PASSED", "details": "Validation passed"}, + ) + + httpx_mock.add_callback( + method="POST", + url=f"{base_url}{org}{tenant}/agentsruntime_/api/execution/guardrails/validate", + callback=capture_request, + ) + + guardrail = BuiltInValidatorGuardrail( + id="test-id", + name="PII detection guardrail", + enabled_for_evals=True, + selector=GuardrailSelector(scopes=[GuardrailScope.LLM]), + guardrail_type="builtInValidator", + validator_type="pii_detection", + byo_validator_name="stray_name", + validator_parameters=[], + ) + + service.evaluate_guardrail("some input", guardrail) + + assert captured_request is not None + request_payload = json.loads(captured_request.content) + assert "byoValidatorName" not in request_payload + + def test_evaluate_guardrail_non_byo_type_never_forwards_byo_fields( + self, + httpx_mock: HTTPXMock, + service: GuardrailsService, + base_url: str, + org: str, + tenant: str, + ) -> None: + """BYO fields are only forwarded for the "byo" sentinel, never leaked + into a non-BYOG validator payload even if a name happens to be set.""" + captured_request = None + + def capture_request(request): + nonlocal captured_request + captured_request = request + return httpx.Response( + status_code=200, + json={"result": "PASSED", "details": "Validation passed"}, + ) + + httpx_mock.add_callback( + method="POST", + url=f"{base_url}{org}{tenant}/agentsruntime_/api/execution/guardrails/validate", + callback=capture_request, + ) + + guardrail = BuiltInValidatorGuardrail( + id="test-id", + name="PII detection guardrail", + enabled_for_evals=True, + selector=GuardrailSelector(scopes=[GuardrailScope.LLM]), + guardrail_type="builtInValidator", + validator_type="pii_detection", + byo_validator_name="stray-name", + validator_parameters=[], + ) + + service.evaluate_guardrail("some input", guardrail) + + assert captured_request is not None + request_payload = json.loads(captured_request.content) + assert "byoValidatorName" not in request_payload + assert "byoConnectionId" not in request_payload + + def test_evaluate_guardrail_ootb_omits_byo_validator_name( + self, + httpx_mock: HTTPXMock, + service: GuardrailsService, + base_url: str, + org: str, + tenant: str, + ) -> None: + """OOTB validators (no byo_validator_name) keep their payload unchanged — + byoValidatorName is not sent.""" + captured_request = None + + def capture_request(request): + nonlocal captured_request + captured_request = request + return httpx.Response( + status_code=200, + json={"result": "PASSED", "details": "Validation passed"}, + ) + + httpx_mock.add_callback( + method="POST", + url=f"{base_url}{org}{tenant}/agentsruntime_/api/execution/guardrails/validate", + callback=capture_request, + ) + + pii_guardrail = BuiltInValidatorGuardrail( + id="test-id", + name="PII detection guardrail", + description="Test PII detection", + enabled_for_evals=True, + selector=GuardrailSelector(scopes=[GuardrailScope.LLM]), + guardrail_type="builtInValidator", + validator_type="pii_detection", + validator_parameters=[], + ) + + service.evaluate_guardrail("some input", pii_guardrail) + + assert captured_request is not None + request_payload = json.loads(captured_request.content) + assert "byoValidatorName" not in request_payload + assert pii_guardrail.byo_validator_name is None + + def test_evaluate_guardrail_sends_trace_context_headers( + self, + httpx_mock: HTTPXMock, + service: GuardrailsService, + base_url: str, + org: str, + tenant: str, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Outgoing request includes trace context headers.""" + captured_request = None + + def capture_request(request): + nonlocal captured_request + captured_request = request + return httpx.Response( + status_code=200, + json={ + "result": "PASSED", + "details": "OK", + }, + ) + + httpx_mock.add_callback( + method="POST", + url=f"{base_url}{org}{tenant}/agentsruntime_/api/execution/guardrails/validate", + callback=capture_request, + ) + + pii_guardrail = BuiltInValidatorGuardrail( + id="test-id", + name="PII guardrail", + description="Test", + enabled_for_evals=True, + selector=GuardrailSelector( + scopes=[GuardrailScope.TOOL], match_names=["tool1"] + ), + guardrail_type="builtInValidator", + validator_type="pii_detection", + validator_parameters=[], + ) + + service.evaluate_guardrail("test input", pii_guardrail) + + assert captured_request is not None + # build_trace_context_headers() injects traceparent/tracestate when + # an active span exists; at minimum, the merge with spec.headers + # should not fail and the request should go through successfully. + # When there IS an active trace context, headers are present: + headers = dict(captured_request.headers) + # The request should have been sent (basic smoke check that + # header merging works even when no active span exists) + assert "content-type" in headers + + def test_evaluate_guardrail_sends_source_and_job_key_headers( + self, + httpx_mock: HTTPXMock, + service: GuardrailsService, + base_url: str, + org: str, + tenant: str, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Outgoing request includes execution source and job key headers.""" + monkeypatch.setenv("UIPATH_JOB_KEY", "job-123") + + captured_request = None + + def capture_request(request): + nonlocal captured_request + captured_request = request + return httpx.Response( + status_code=200, + json={"result": "PASSED", "details": "OK"}, + ) + + httpx_mock.add_callback( + method="POST", + url=f"{base_url}{org}{tenant}/agentsruntime_/api/execution/guardrails/validate", + callback=capture_request, + ) + + pii_guardrail = BuiltInValidatorGuardrail( + id="test-id", + name="PII guardrail", + description="Test", + enabled_for_evals=True, + selector=GuardrailSelector( + scopes=[GuardrailScope.TOOL], match_names=["tool1"] + ), + guardrail_type="builtInValidator", + validator_type="pii_detection", + validator_parameters=[], + ) + + with ExecutionSourceContext("runtime"): + service.evaluate_guardrail("test input", pii_guardrail) + + assert captured_request is not None + headers = dict(captured_request.headers) + assert headers.get("x-uipath-guardrails-source") == "runtime" + assert headers.get("x-uipath-jobkey") == "job-123" + + def test_evaluate_guardrail_omits_source_and_job_key_when_unset( + self, + httpx_mock: HTTPXMock, + service: GuardrailsService, + base_url: str, + org: str, + tenant: str, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Source/job key headers are absent when unset.""" + monkeypatch.delenv("UIPATH_JOB_KEY", raising=False) + + captured_request = None + + def capture_request(request): + nonlocal captured_request + captured_request = request + return httpx.Response( + status_code=200, + json={"result": "PASSED", "details": "OK"}, + ) + + httpx_mock.add_callback( + method="POST", + url=f"{base_url}{org}{tenant}/agentsruntime_/api/execution/guardrails/validate", + callback=capture_request, + ) + + pii_guardrail = BuiltInValidatorGuardrail( + id="test-id", + name="PII guardrail", + description="Test", + enabled_for_evals=True, + selector=GuardrailSelector( + scopes=[GuardrailScope.TOOL], match_names=["tool1"] + ), + guardrail_type="builtInValidator", + validator_type="pii_detection", + validator_parameters=[], + ) + + service.evaluate_guardrail("test input", pii_guardrail) + + assert captured_request is not None + headers = dict(captured_request.headers) + assert "x-uipath-guardrails-source" not in headers + assert "x-uipath-jobkey" not in headers + + def test_evaluate_guardrail_extracts_span_id_from_traceparent( + self, + httpx_mock: HTTPXMock, + service: GuardrailsService, + base_url: str, + org: str, + tenant: str, + ) -> None: + """Response with x-uipath-traceparent-id header populates span_id.""" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/agentsruntime_/api/execution/guardrails/validate", + status_code=200, + json={ + "result": "VALIDATION_FAILED", + "details": "PII detected", + }, + headers={ + "x-uipath-traceparent-id": "00-abcdef1234567890abcdef1234567890-1234567890abcdef" + }, + ) + + pii_guardrail = BuiltInValidatorGuardrail( + id="test-id", + name="PII guardrail", + description="Test", + enabled_for_evals=True, + selector=GuardrailSelector( + scopes=[GuardrailScope.TOOL], match_names=["tool1"] + ), + guardrail_type="builtInValidator", + validator_type="pii_detection", + validator_parameters=[], + ) + + result = service.evaluate_guardrail("test input", pii_guardrail) + + assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.span_id == "00000000-0000-0000-1234-567890abcdef" + + def test_evaluate_guardrail_no_traceparent_header_no_span_id( + self, + httpx_mock: HTTPXMock, + service: GuardrailsService, + base_url: str, + org: str, + tenant: str, + ) -> None: + """Response without x-uipath-traceparent-id header leaves span_id as None.""" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/agentsruntime_/api/execution/guardrails/validate", + status_code=200, + json={ + "result": "PASSED", + "details": "OK", + }, + ) + + pii_guardrail = BuiltInValidatorGuardrail( + id="test-id", + name="PII guardrail", + description="Test", + enabled_for_evals=True, + selector=GuardrailSelector( + scopes=[GuardrailScope.TOOL], match_names=["tool1"] + ), + guardrail_type="builtInValidator", + validator_type="pii_detection", + validator_parameters=[], + ) + + result = service.evaluate_guardrail("test input", pii_guardrail) + + assert result.result == GuardrailValidationResultType.PASSED + assert result.span_id is None + + class TestExtractSpanIdFromTraceparent: + """Tests for _extract_span_id_from_traceparent.""" + + def test_valid_traceparent_16_char_span_id(self) -> None: + result = GuardrailsService._extract_span_id_from_traceparent( + "00-abcdef1234567890abcdef1234567890-1234567890abcdef" + ) + assert result == "00000000-0000-0000-1234-567890abcdef" + + def test_valid_traceparent_32_char_span_id(self) -> None: + result = GuardrailsService._extract_span_id_from_traceparent( + "00-abcdef1234567890abcdef1234567890-0a1b2c3d4e5f67890a1b2c3d4e5f6789" + ) + assert result == "0a1b2c3d-4e5f-6789-0a1b-2c3d4e5f6789" + + def test_none_input(self) -> None: + assert GuardrailsService._extract_span_id_from_traceparent(None) is None + + def test_empty_string(self) -> None: + assert GuardrailsService._extract_span_id_from_traceparent("") is None + + def test_valid_traceparent_4_part_with_trace_flags(self) -> None: + result = GuardrailsService._extract_span_id_from_traceparent( + "00-abcdef1234567890abcdef1234567890-1234567890abcdef-01" + ) + assert result == "00000000-0000-0000-1234-567890abcdef" + + def test_uppercase_hex_normalized_to_lowercase(self) -> None: + result = GuardrailsService._extract_span_id_from_traceparent( + "00-ABCDEF1234567890ABCDEF1234567890-1234567890ABCDEF" + ) + assert result == "00000000-0000-0000-1234-567890abcdef" + + def test_invalid_span_id_length_rejected(self) -> None: + """Span IDs that are neither 16 nor 32 hex chars are rejected.""" + assert ( + GuardrailsService._extract_span_id_from_traceparent( + "00-abcdef1234567890abcdef1234567890-1234abcd" + ) + is None + ) + + def test_invalid_format(self) -> None: + assert ( + GuardrailsService._extract_span_id_from_traceparent("not-valid") is None + ) diff --git a/packages/uipath-platform/tests/services/test_hitl.py b/packages/uipath-platform/tests/services/test_hitl.py index aa288137d..f0c463251 100644 --- a/packages/uipath-platform/tests/services/test_hitl.py +++ b/packages/uipath-platform/tests/services/test_hitl.py @@ -1,12 +1,16 @@ +import json import uuid +from datetime import datetime, timedelta, timezone from typing import Any from unittest.mock import AsyncMock, patch import pytest from pytest_httpx import HTTPXMock from uipath.core.errors import ErrorCategory, UiPathFaultedTriggerError +from uipath.core.serialization import serialize_object from uipath.core.triggers import ( UiPathApiTrigger, + UiPathIntegrationTrigger, UiPathResumeTrigger, UiPathResumeTriggerName, UiPathResumeTriggerType, @@ -32,11 +36,14 @@ WaitDocumentExtractionValidation, WaitEphemeralIndex, WaitEphemeralIndexRaw, + WaitIntegrationEvent, WaitJob, WaitJobRaw, WaitSystemAgent, WaitTask, + WaitUntil, ) +from uipath.platform.connections import Connection from uipath.platform.context_grounding import ( BatchTransformCreationResponse, BatchTransformOutputColumn, @@ -60,6 +67,7 @@ StartExtractionValidationResponse, ValidateExtractionAction, ) +from uipath.platform.errors import ContextGroundingIndexNotFoundError from uipath.platform.orchestrator import Job, JobErrorInfo from uipath.platform.orchestrator.job import JobState from uipath.platform.resume_triggers import ( @@ -508,6 +516,91 @@ async def test_read_api_trigger_failure( await reader.read_trigger(resume_trigger) assert exc_info.value.category == ErrorCategory.SYSTEM + @pytest.mark.anyio + async def test_read_inbox_trigger( + self, + httpx_mock: HTTPXMock, + base_url: str, + setup_test_env: None, + ) -> None: + """Test reading an Inbox trigger fetches the IS metadata via GetPayload + and then enriches it via /elements_/v1/events/{processedEventId}. + """ + inbox_id = str(uuid.uuid4()) + processed_event_id = "v2::pp::1777041494382::334071::e374ecd5d0f73c21" + inbox_metadata = { + "UiPathEventConnector": "uipath-slack", + "UiPathEvent": "NEW_MESSAGE", + "UiPathEventObjectType": "Message", + "UiPathEventObjectId": "C123:1777041494.382", + "UiPathAdditionalEventData": json.dumps( + {"processedEventId": processed_event_id} + ), + } + enriched_event = { + "channel": "alerts", + "user": "U456", + "text": "hello from slack", + "ts": "1777041494.382", + } + + httpx_mock.add_response( + url=f"{base_url}/orchestrator_/api/JobTriggers/GetPayload/{inbox_id}", + status_code=200, + json=inbox_metadata, + ) + httpx_mock.add_response( + url=f"{base_url}/elements_/v1/events/{processed_event_id}", + status_code=200, + json=enriched_event, + ) + + resume_trigger = UiPathResumeTrigger( + trigger_type=UiPathResumeTriggerType.INBOX, + integration_resume=UiPathIntegrationTrigger( + connector="slack", + connection_id=str(uuid.uuid4()), + operation="OnMessage", + object_name="Message", + inbox_id=inbox_id, + ), + ) + + reader = UiPathResumeTriggerReader() + result = await reader.read_trigger(resume_trigger) + assert result == enriched_event + + @pytest.mark.anyio + async def test_read_inbox_trigger_failure( + self, + httpx_mock: HTTPXMock, + base_url: str, + setup_test_env: None, + ) -> None: + """Test reading an Inbox trigger with a failed payload response.""" + inbox_id = str(uuid.uuid4()) + + httpx_mock.add_response( + url=f"{base_url}/orchestrator_/api/JobTriggers/GetPayload/{inbox_id}", + status_code=500, + ) + + resume_trigger = UiPathResumeTrigger( + trigger_type=UiPathResumeTriggerType.INBOX, + integration_resume=UiPathIntegrationTrigger( + connector="slack", + connection_id=str(uuid.uuid4()), + operation="OnMessage", + object_name="Message", + inbox_id=inbox_id, + ), + ) + + with pytest.raises(UiPathFaultedTriggerError) as exc_info: + reader = UiPathResumeTriggerReader() + await reader.read_trigger(resume_trigger) + assert exc_info.value.category == ErrorCategory.SYSTEM + @pytest.mark.anyio async def test_read_deep_rag_trigger_successful( self, @@ -809,6 +902,41 @@ async def test_read_batch_rag_trigger_pending( reader = UiPathResumeTriggerReader() await reader.read_trigger(resume_trigger) + @pytest.mark.anyio + async def test_read_batch_rag_trigger_failed( + self, + setup_test_env: None, + ) -> None: + """Test reading a failed batch rag trigger raises faulted error.""" + from uipath.core.errors import UiPathFaultedTriggerError + + from uipath.platform.errors import BatchTransformFailedException + + task_id = "test-batch-rag-id" + destination_path = "test/output.xlsx" + mock_download_async = AsyncMock( + side_effect=BatchTransformFailedException(task_id) + ) + + with patch( + "uipath.platform.context_grounding._context_grounding_service.ContextGroundingService.download_batch_transform_result_async", + new=mock_download_async, + ): + resume_trigger = UiPathResumeTrigger( + trigger_type=UiPathResumeTriggerType.BATCH_RAG, + item_key=task_id, + folder_key="test-folder", + folder_path="test-path", + payload={ + "index_name": "test-index", + "destination_path": destination_path, + }, + ) + + with pytest.raises(UiPathFaultedTriggerError): + reader = UiPathResumeTriggerReader() + await reader.read_trigger(resume_trigger) + @pytest.mark.anyio async def test_read_ephemeral_index_trigger_successful( self, @@ -1090,6 +1218,21 @@ async def test_read_ixp_vs_escalation_trigger_unassigned( reader = UiPathResumeTriggerReader() await reader.read_trigger(resume_trigger) + @pytest.mark.anyio + async def test_read_timer_trigger_serializes_resume_time(self) -> None: + """Test reading a timer trigger returns JSON-safe resume time data.""" + resume_time = datetime(2026, 6, 27, 20, 14, 49, tzinfo=timezone.utc) + resume_trigger = UiPathResumeTrigger( + trigger_type=UiPathResumeTriggerType.TIMER, + trigger_name=UiPathResumeTriggerName.TIMER, + resume_time=resume_time, + ) + + reader = UiPathResumeTriggerReader() + result = await reader.read_trigger(resume_trigger) + + assert result == {"resumeTime": serialize_object(resume_time)} + class TestHitlProcessor: """Tests for the HitlProcessor class.""" @@ -1339,6 +1482,186 @@ async def test_create_resume_trigger_api( assert isinstance(resume_trigger.api_resume.inbox_id, str) assert resume_trigger.api_resume.request == api_input + @pytest.mark.anyio + async def test_create_resume_trigger_wait_integration_event( + self, + setup_test_env: None, + ) -> None: + """Test creating a resume trigger for WaitIntegrationEvent.""" + connection_id = str(uuid.uuid4()) + mock_connection = Connection( + id=connection_id, name="Slack-Alerts", element_instance_id=1 + ) + mock_list_async = AsyncMock(return_value=[mock_connection]) + + wait_event = WaitIntegrationEvent( + connector="slack", + connection_name="Slack-Alerts", + connection_folder_path="Shared", + operation="OnMessage", + object_name="Message", + filter_expression="channel == 'alerts'", + parameters={"channel_id": "C123"}, + ) + + with patch( + "uipath.platform.connections._connections_service.ConnectionsService.list_async", + new=mock_list_async, + ): + processor = UiPathResumeTriggerCreator() + resume_trigger = await processor.create_trigger(wait_event) + + assert resume_trigger is not None + assert resume_trigger.trigger_type == UiPathResumeTriggerType.INBOX + assert resume_trigger.trigger_name == UiPathResumeTriggerName.INBOX + assert resume_trigger.api_resume is None + assert resume_trigger.integration_resume is not None + assert resume_trigger.integration_resume.connector == "slack" + assert resume_trigger.integration_resume.connection_id == connection_id + assert resume_trigger.integration_resume.operation == "OnMessage" + assert resume_trigger.integration_resume.object_name == "Message" + assert ( + resume_trigger.integration_resume.filter_expression == "channel == 'alerts'" + ) + assert resume_trigger.integration_resume.parameters == {"channel_id": "C123"} + assert isinstance(resume_trigger.integration_resume.inbox_id, str) + uuid.UUID(resume_trigger.integration_resume.inbox_id) + mock_list_async.assert_called_once_with( + name="Slack-Alerts", folder_path="Shared", connector_key="slack" + ) + + @pytest.mark.anyio + async def test_create_resume_trigger_wait_integration_event_optional_fields_omitted( + self, + setup_test_env: None, + ) -> None: + """Test that filter_expression, parameters, and folder_path are optional.""" + mock_connection = Connection( + id=str(uuid.uuid4()), name="Teams-Default", element_instance_id=2 + ) + mock_list_async = AsyncMock(return_value=[mock_connection]) + + wait_event = WaitIntegrationEvent( + connector="teams", + connection_name="Teams-Default", + operation="OnReply", + object_name="Reply", + ) + + with patch( + "uipath.platform.connections._connections_service.ConnectionsService.list_async", + new=mock_list_async, + ): + processor = UiPathResumeTriggerCreator() + resume_trigger = await processor.create_trigger(wait_event) + + assert resume_trigger.integration_resume is not None + assert resume_trigger.integration_resume.filter_expression is None + assert resume_trigger.integration_resume.parameters is None + mock_list_async.assert_called_once_with( + name="Teams-Default", folder_path=None, connector_key="teams" + ) + + @pytest.mark.anyio + async def test_wait_integration_event_serializes_with_camelcase_aliases( + self, + setup_test_env: None, + ) -> None: + """Wire shape: UiPathResumeTrigger.integration_resume must serialize + with the field names Orchestrator's ResumeTriggerDto/IntegrationResumeDto + expect (PascalCase-ish camelCase). + """ + connection_id = str(uuid.uuid4()) + mock_connection = Connection( + id=connection_id, name="Slack-Alerts", element_instance_id=3 + ) + mock_list_async = AsyncMock(return_value=[mock_connection]) + + wait_event = WaitIntegrationEvent( + connector="slack", + connection_name="Slack-Alerts", + operation="OnMessage", + object_name="Message", + ) + + with patch( + "uipath.platform.connections._connections_service.ConnectionsService.list_async", + new=mock_list_async, + ): + processor = UiPathResumeTriggerCreator() + resume_trigger = await processor.create_trigger(wait_event) + + dumped = resume_trigger.model_dump(by_alias=True, exclude_none=True) + + assert dumped["triggerType"] == UiPathResumeTriggerType.INBOX + assert "integrationResume" in dumped + integration = dumped["integrationResume"] + assert integration["connector"] == "slack" + assert integration["connectionId"] == connection_id + assert integration["operation"] == "OnMessage" + assert integration["objectName"] == "Message" + assert "inboxId" in integration + uuid.UUID(integration["inboxId"]) + + @pytest.mark.anyio + async def test_create_resume_trigger_wait_integration_event_no_match( + self, + setup_test_env: None, + ) -> None: + """Listing returns no exact-name match -> creator raises.""" + mock_list_async = AsyncMock(return_value=[]) + + wait_event = WaitIntegrationEvent( + connector="slack", + connection_name="Missing-Connection", + operation="OnMessage", + object_name="Message", + ) + + with patch( + "uipath.platform.connections._connections_service.ConnectionsService.list_async", + new=mock_list_async, + ): + processor = UiPathResumeTriggerCreator() + with pytest.raises(UiPathFaultedTriggerError): + await processor.create_trigger(wait_event) + + @pytest.mark.anyio + async def test_create_resume_trigger_wait_integration_event_filters_to_exact_match( + self, + setup_test_env: None, + ) -> None: + """list_async partial-matches; creator must pick the exact-name entry.""" + target_id = str(uuid.uuid4()) + # list_async partial-matches; simulate prefix-matching returning extras + mock_list_async = AsyncMock( + return_value=[ + Connection( + id=str(uuid.uuid4()), + name="Slack-Alerts-Old", + element_instance_id=4, + ), + Connection(id=target_id, name="Slack-Alerts", element_instance_id=5), + ] + ) + + wait_event = WaitIntegrationEvent( + connector="slack", + connection_name="Slack-Alerts", + operation="OnMessage", + object_name="Message", + ) + + with patch( + "uipath.platform.connections._connections_service.ConnectionsService.list_async", + new=mock_list_async, + ): + processor = UiPathResumeTriggerCreator() + resume_trigger = await processor.create_trigger(wait_event) + + assert resume_trigger.integration_resume is not None + assert resume_trigger.integration_resume.connection_id == target_id + @pytest.mark.anyio async def test_create_resume_trigger_create_deep_rag( self, @@ -1383,6 +1706,38 @@ async def test_create_resume_trigger_create_deep_rag( folder_key=create_deep_rag.index_folder_key, ) + @pytest.mark.anyio + async def test_missing_deep_rag_index_is_deployment_error( + self, + setup_test_env: None, + ) -> None: + create_deep_rag = CreateDeepRag( + name="test-deep-rag", + index_name="Files", + prompt="test prompt", + glob_pattern="**/*.pdf", + citation_mode=CitationMode.INLINE, + index_folder_path="/test/path", + ) + missing_index = ContextGroundingIndexNotFoundError("Files") + mock_start_deep_rag = AsyncMock(side_effect=missing_index) + + with patch( + "uipath.platform.context_grounding._context_grounding_service.ContextGroundingService.start_deep_rag_async", + new=mock_start_deep_rag, + ): + with pytest.raises(UiPathFaultedTriggerError) as exc_info: + await UiPathResumeTriggerCreator().create_trigger(create_deep_rag) + + error = exc_info.value + assert error.category == ErrorCategory.DEPLOYMENT + assert error.message == ( + "Context grounding index not found. Check that the index is deployed and " + "available in the configured folder." + ) + assert error.detail == "ContextGroundingIndex 'Files' not found" + assert error.__cause__ is missing_index + @pytest.mark.anyio async def test_create_resume_trigger_wait_deep_rag( self, @@ -1510,6 +1865,42 @@ async def test_create_resume_trigger_create_batch_transform( folder_key=create_batch_transform.index_folder_key, ) + @pytest.mark.anyio + async def test_missing_batch_transform_index_is_deployment_error( + self, + setup_test_env: None, + ) -> None: + create_batch_transform = CreateBatchTransform( + name="test-batch-transform", + index_name="Files", + prompt="test prompt", + output_columns=[ + BatchTransformOutputColumn(name="column1", description="desc1") + ], + destination_path="/output/path.xlsx", + index_folder_path="/test/path", + ) + missing_index = ContextGroundingIndexNotFoundError("Files") + mock_start_batch_transform = AsyncMock(side_effect=missing_index) + + with patch( + "uipath.platform.context_grounding._context_grounding_service.ContextGroundingService.start_batch_transform_async", + new=mock_start_batch_transform, + ): + with pytest.raises(UiPathFaultedTriggerError) as exc_info: + await UiPathResumeTriggerCreator().create_trigger( + create_batch_transform + ) + + error = exc_info.value + assert error.category == ErrorCategory.DEPLOYMENT + assert error.message == ( + "Context grounding index not found. Check that the index is deployed and " + "available in the configured folder." + ) + assert error.detail == "ContextGroundingIndex 'Files' not found" + assert error.__cause__ is missing_index + @pytest.mark.anyio async def test_create_resume_trigger_wait_batch_transform( self, @@ -1759,6 +2150,73 @@ async def test_create_resume_trigger_wait_document_extraction_validation( assert resume_trigger.trigger_type == UiPathResumeTriggerType.IXP_VS_ESCALATION assert resume_trigger.item_key == operation_id + @pytest.mark.anyio + async def test_create_resume_trigger_wait_until_normalizes_to_utc(self) -> None: + """Test creating a timer resume trigger for WaitUntil.""" + local_resume_time = datetime( + 2026, + 6, + 27, + 23, + 14, + 49, + tzinfo=timezone(timedelta(hours=3)), + ) + wait_until = WaitUntil(resume_time=local_resume_time) + + processor = UiPathResumeTriggerCreator() + resume_trigger = await processor.create_trigger(wait_until) + + assert resume_trigger.trigger_type == UiPathResumeTriggerType.TIMER + assert resume_trigger.trigger_name == UiPathResumeTriggerName.TIMER + assert resume_trigger.resume_time == datetime( + 2026, 6, 27, 20, 14, 49, tzinfo=timezone.utc + ) + + def test_wait_until_requires_timezone_aware_resume_time(self) -> None: + """Test WaitUntil rejects timezone-naive resume times.""" + with pytest.raises(ValueError, match="resume_time must include timezone"): + WaitUntil(resume_time=datetime(2026, 6, 27, 20, 14, 49)) + + @pytest.mark.anyio + async def test_create_resume_triggers_for_interrupt_list( + self, + ) -> None: + """Test an interrupt list creates sibling triggers for the same interrupt.""" + job_key = "test-job-key" + wait_job = WaitJob( + job=Job( + id=1234, + key=job_key, + folder_key="d0e09040-5997-44e1-93b7-4087689521b7", + ), + process_folder_path="/test/path", + ) + wait_until = WaitUntil( + resume_time=datetime(2026, 6, 27, 23, 14, 49, tzinfo=timezone.utc) + ) + + processor = UiPathResumeTriggerCreator() + triggers = await processor.create_triggers([wait_job, wait_until]) + + assert len(triggers) == 2 + job_trigger, timer_trigger = triggers + assert job_trigger.trigger_type == UiPathResumeTriggerType.JOB + assert job_trigger.item_key == job_key + assert timer_trigger.trigger_type == UiPathResumeTriggerType.TIMER + assert timer_trigger.trigger_name == UiPathResumeTriggerName.TIMER + assert timer_trigger.resume_time == datetime( + 2026, 6, 27, 23, 14, 49, tzinfo=timezone.utc + ) + + @pytest.mark.anyio + async def test_create_resume_triggers_rejects_empty_interrupt_list(self) -> None: + """Test an interrupt list must include at least one model.""" + processor = UiPathResumeTriggerCreator() + + with pytest.raises(ValueError, match="At least one interrupt model"): + await processor.create_triggers([]) + class TestDocumentExtractionModels: """Tests for document extraction models.""" diff --git a/packages/uipath-platform/tests/services/test_jobs_service.py b/packages/uipath-platform/tests/services/test_jobs_service.py index e0321717e..6782855d4 100644 --- a/packages/uipath-platform/tests/services/test_jobs_service.py +++ b/packages/uipath-platform/tests/services/test_jobs_service.py @@ -9,7 +9,7 @@ from pytest_mock import MockerFixture from uipath.platform import UiPathApiConfig, UiPathExecutionContext -from uipath.platform.common.constants import HEADER_USER_AGENT, TEMP_ATTACHMENTS_FOLDER +from uipath.platform.constants import HEADER_USER_AGENT, TEMP_ATTACHMENTS_FOLDER from uipath.platform.orchestrator import Job from uipath.platform.orchestrator._jobs_service import JobsService @@ -101,6 +101,15 @@ def local_attachment_file( class TestJobsService: + @pytest.mark.anyio + async def test_aclose_closes_owned_services(self, service: JobsService): + await service.aclose() + + assert service._client.is_closed + assert service._client_async.is_closed + assert service._attachments_service._client.is_closed + assert service._attachments_service._client_async.is_closed + def test_retrieve( self, httpx_mock: HTTPXMock, diff --git a/packages/uipath-platform/tests/services/test_live_track_event_dispatcher.py b/packages/uipath-platform/tests/services/test_live_track_event_dispatcher.py new file mode 100644 index 000000000..8e6b0ebb7 --- /dev/null +++ b/packages/uipath-platform/tests/services/test_live_track_event_dispatcher.py @@ -0,0 +1,581 @@ +"""Tests for LiveTrackEventDispatcher. + +The dispatcher schedules ``provider.track_event_async`` on a private +background asyncio loop so a sync caller never blocks on the +underlying HTTP. Tests focus on: + +- ``dispatch`` returns immediately (doesn't block on the coroutine) +- The provider's ``track_event_async`` is awaited with the same kwargs +- Exceptions in the coroutine are swallowed (fire-and-forget contract) +- Multiple concurrent submissions all reach the provider +- ``shutdown`` drains pending coroutines +- Post-shutdown dispatch is silent and does not call the provider +- Saturated in-flight cap drops submissions rather than queueing +""" + +from __future__ import annotations + +import asyncio +import logging +import threading +import time +from collections.abc import Generator +from unittest.mock import MagicMock + +import pytest + +from uipath.platform.governance import UiPathPlatformGovernanceProvider +from uipath.platform.governance._live_track_event_dispatcher import ( + LiveTrackEventDispatcher, +) + +_DISPATCHER_MODULE = "uipath.platform.governance._live_track_event_dispatcher" +_DISPATCHER_LOGGER = _DISPATCHER_MODULE + + +@pytest.fixture +def provider() -> MagicMock: + """Mock provider — ``track_event_async`` becomes an ``AsyncMock`` via spec.""" + # ``MagicMock(spec=...)`` auto-detects coroutine functions on the spec + # and creates ``AsyncMock`` attributes for them, so + # ``provider.track_event_async(...)`` returns an awaitable. + return MagicMock(spec=UiPathPlatformGovernanceProvider) + + +@pytest.fixture +def dispatcher( + provider: MagicMock, +) -> Generator[LiveTrackEventDispatcher, None, None]: + """Dispatcher with a small in-flight cap for fast tests.""" + d = LiveTrackEventDispatcher(provider, max_inflight=4) + yield d + d.shutdown() + + +def _wait_for(predicate, *, timeout: float = 2.0, interval: float = 0.01) -> bool: + """Spin-wait helper — returns True when predicate passes, False on timeout.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(interval) + return False + + +# --------------------------------------------------------------------------- +# dispatch is non-blocking +# --------------------------------------------------------------------------- + + +def test_dispatch_returns_before_provider_completes( + provider: MagicMock, + dispatcher: LiveTrackEventDispatcher, +) -> None: + """dispatch must not wait for the coroutine — the calling thread must not block.""" + started = threading.Event() + + async def _slow_track_event(**_: object) -> None: + started.set() + # Sleep long enough that a blocking dispatch would fail the timing bound. + await asyncio.sleep(0.5) + + provider.track_event_async.side_effect = _slow_track_event + + t0 = time.monotonic() + dispatcher.dispatch(event_name="agent.started") + elapsed = time.monotonic() - t0 + + # dispatch should return in well under 100ms even though the coroutine + # is sleeping for 500ms. + assert elapsed < 0.1, f"dispatch blocked for {elapsed:.3f}s" + + # Coroutine did start (proves the submission landed on the loop). + assert started.wait(timeout=2.0), "coroutine never started" + + +# --------------------------------------------------------------------------- +# provider receives the exact kwargs +# --------------------------------------------------------------------------- + + +def test_dispatch_forwards_kwargs_to_provider( + provider: MagicMock, + dispatcher: LiveTrackEventDispatcher, +) -> None: + """The dispatcher is a thin adapter — every kwarg must reach track_event_async.""" + dispatcher.dispatch( + event_name="agent.tool_call", + data={"tool": "browser.open", "url": "https://example.com"}, + operation_id="op-abc-123", + ) + + # Wait for the coroutine to be awaited. + assert _wait_for(lambda: provider.track_event_async.await_count >= 1), ( + "track_event_async never awaited" + ) + + provider.track_event_async.assert_awaited_once_with( + event_name="agent.tool_call", + data={"tool": "browser.open", "url": "https://example.com"}, + operation_id="op-abc-123", + ) + + +def test_dispatch_passes_none_data_and_operation_id( + provider: MagicMock, + dispatcher: LiveTrackEventDispatcher, +) -> None: + """Defaults — ``data`` and ``operation_id`` flow through as ``None``.""" + dispatcher.dispatch(event_name="agent.idle") + + assert _wait_for(lambda: provider.track_event_async.await_count >= 1) + provider.track_event_async.assert_awaited_once_with( + event_name="agent.idle", + data=None, + operation_id=None, + ) + + +# --------------------------------------------------------------------------- +# exceptions are swallowed +# --------------------------------------------------------------------------- + + +def test_worker_exception_does_not_propagate( + provider: MagicMock, + dispatcher: LiveTrackEventDispatcher, +) -> None: + """Fire-and-forget — dispatch returns before the coroutine runs, so an + exception raised inside it cannot reach the caller. The dispatcher + must catch and log internally rather than letting the future + finalize with an unobserved exception. + """ + provider.track_event_async.side_effect = RuntimeError("simulated backend 5xx") + + # If the dispatcher leaked the exception, this call would raise. + dispatcher.dispatch(event_name="agent.deny") + + # And subsequent calls keep working — one bad event doesn't poison + # the loop. + provider.track_event_async.side_effect = None + dispatcher.dispatch(event_name="agent.deny") + + assert _wait_for(lambda: provider.track_event_async.await_count >= 2) + assert provider.track_event_async.await_count == 2 + + +# --------------------------------------------------------------------------- +# concurrency +# --------------------------------------------------------------------------- + + +def test_multiple_dispatches_all_reach_provider(provider: MagicMock) -> None: + """A burst of submissions must all be delivered, in any order.""" + # Use a dedicated dispatcher with headroom well above the burst + # size so the fast-burst doesn't race the loop into saturation. + dispatcher = LiveTrackEventDispatcher(provider, max_inflight=64) + try: + for i in range(20): + dispatcher.dispatch(event_name=f"event.{i}") + + assert _wait_for(lambda: provider.track_event_async.await_count == 20) + + seen = { + call.kwargs["event_name"] + for call in provider.track_event_async.await_args_list + } + assert seen == {f"event.{i}" for i in range(20)} + finally: + dispatcher.shutdown() + + +# --------------------------------------------------------------------------- +# shutdown +# --------------------------------------------------------------------------- + + +def test_shutdown_waits_for_pending(provider: MagicMock) -> None: + """``shutdown(wait=True)`` must let in-flight coroutines finish before + returning so process teardown doesn't lose telemetry. + """ + completed: list[str] = [] + + async def _record(*, event_name: str, **_: object) -> None: + # Small await so submissions overlap the shutdown call. + await asyncio.sleep(0.05) + completed.append(event_name) + + provider.track_event_async.side_effect = _record + dispatcher = LiveTrackEventDispatcher(provider, max_inflight=10) + + for i in range(5): + dispatcher.dispatch(event_name=f"event.{i}") + + dispatcher.shutdown(wait=True) + + # Every submission ran to completion by the time shutdown returned. + assert sorted(completed) == [f"event.{i}" for i in range(5)] + + +def test_shutdown_is_idempotent(provider: MagicMock) -> None: + """Calling shutdown twice must not raise — process teardown paths + sometimes invoke close/shutdown from multiple atexit hooks. + """ + dispatcher = LiveTrackEventDispatcher(provider, max_inflight=4) + dispatcher.shutdown() + dispatcher.shutdown() # second call: no crash, no exception + + +# --------------------------------------------------------------------------- +# fire-and-forget safety: dispatch never raises +# --------------------------------------------------------------------------- + + +def test_dispatch_after_shutdown_is_silent(provider: MagicMock) -> None: + """After :meth:`shutdown` the dispatcher must silently drop late + dispatches — a late dispatch (e.g. from an atexit cleanup after + the loop already stopped) cannot be allowed to raise. + """ + dispatcher = LiveTrackEventDispatcher(provider, max_inflight=4) + dispatcher.shutdown(wait=True) + + # If the dispatcher leaked an exception, this call would raise. + dispatcher.dispatch(event_name="agent.late") + + # And the provider must not have been awaited — the loop is down. + assert not provider.track_event_async.await_count + + +def test_shutdown_no_wait_cancels_inflight_cleanly(provider: MagicMock) -> None: + """``shutdown(wait=False)`` while coroutines are mid-await must cancel + them and let :meth:`_on_future_done` complete its accounting on the + cancellation path — no semaphore leak, no leftover future in the + pending set. + + Regression: ``concurrent.futures.Future.exception()`` *raises* + ``CancelledError`` when the future was cancelled. If the callback + doesn't wrap the observation in a targeted ``except``, the discard + + release calls are skipped → semaphore slots leak (silently, + because ``Future._invoke_callbacks`` swallows the exception into + the logger). Without this guard the leak isn't visible at a + process-level assertion — the loop still stops — so the test must + check the semaphore state directly. + """ + n = 3 + release = threading.Event() + + async def _hang_forever(**_: object) -> None: + # Yield to the loop but never complete — cancelled at teardown. + while not release.is_set(): + await asyncio.sleep(0.02) + + provider.track_event_async.side_effect = _hang_forever + + dispatcher = LiveTrackEventDispatcher(provider, max_inflight=n) + try: + # Fill every in-flight slot so shutdown teardown will cancel + # every one of them. + for i in range(n): + dispatcher.dispatch(event_name=f"event.hang.{i}") + assert _wait_for(lambda: provider.track_event_async.await_count >= 1) + + # wait=False: don't drain, just stop the loop. The loop's finally + # block cancels the pending tasks; each cancellation triggers + # ``_on_future_done`` on the cancelled ``concurrent.futures.Future``. + dispatcher.shutdown(wait=False) + + # Loop thread joined. + assert not dispatcher._loop_thread.is_alive(), ( + "loop thread did not stop after shutdown(wait=False)" + ) + + # Accounting cleanly reset — the callback ran to its finally + # even on the CancelledError path. + # 1. Pending set drained: every cancelled future was discarded. + assert not dispatcher._futures, ( + f"cancellation left {len(dispatcher._futures)} future(s) in " + f"the pending set — semaphore slot(s) leaked" + ) + # 2. Every in-flight slot released: we can immediately re-acquire + # all ``n`` semaphore slots without blocking. + acquired = 0 + for _ in range(n): + if dispatcher._inflight.acquire(blocking=False): + acquired += 1 + for _ in range(acquired): + dispatcher._inflight.release() + assert acquired == n, ( + f"expected all {n} semaphore slots free after cancellation, " + f"only {acquired} were released" + ) + finally: + release.set() + + +def test_dispatch_drops_when_inflight_saturated(provider: MagicMock) -> None: + """When the in-flight cap is reached, further dispatches are dropped + rather than queueing unboundedly. The drop must NOT call the + provider for the saturated submission. + """ + release = threading.Event() + + async def _blocked(**_: object) -> None: + # Poll a threading.Event without blocking the loop — a bare + # ``release.wait()`` would freeze the whole event loop; the + # small await yields between checks so other coroutines can + # progress and the semaphore state can be inspected. + while not release.is_set(): + await asyncio.sleep(0.02) + + provider.track_event_async.side_effect = _blocked + + dispatcher = LiveTrackEventDispatcher(provider, max_inflight=2) + try: + # Fill the cap. + dispatcher.dispatch(event_name="event.1") + dispatcher.dispatch(event_name="event.2") + + # Wait for at least one coroutine to be awaited (so the + # semaphore is held by an in-flight task, not just queued). + assert _wait_for(lambda: provider.track_event_async.await_count >= 1) + + # Third submission should be dropped — semaphore is exhausted. + dispatcher.dispatch(event_name="event.dropped") + + # event.dropped never reaches the provider. + assert "event.dropped" not in { + call.kwargs.get("event_name") + for call in provider.track_event_async.call_args_list + } + finally: + release.set() + dispatcher.shutdown(wait=True) + + +# --------------------------------------------------------------------------- +# construction defaults +# --------------------------------------------------------------------------- + + +def test_default_max_inflight_matches_module_constant(provider: MagicMock) -> None: + """Constructor default equals the documented module constant. + + Guards against silent drift between the docstring's cited default + and the actual value passed to :class:`BoundedSemaphore`. + """ + dispatcher = LiveTrackEventDispatcher(provider) + try: + assert ( + dispatcher._max_inflight == LiveTrackEventDispatcher._DEFAULT_MAX_INFLIGHT + ) + assert LiveTrackEventDispatcher._DEFAULT_MAX_INFLIGHT == 40 + finally: + dispatcher.shutdown() + + +# --------------------------------------------------------------------------- +# uncovered branches: loop-unavailable during dispatch, loop-already-stopped +# during shutdown, and explicit exception-log path +# --------------------------------------------------------------------------- + + +def test_dispatch_swallows_when_loop_unavailable( + provider: MagicMock, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """Covers the ``RuntimeError`` catch in :meth:`dispatch`. + + Simulates the race where the loop is stopped/closed between the + shutdown-event check and ``run_coroutine_threadsafe``. dispatch + must: + + - not raise + - not call the provider + - release the semaphore slot it took (so subsequent live dispatches + can still acquire) + - log at debug + """ + dispatcher = LiveTrackEventDispatcher(provider, max_inflight=2) + try: + # Force run_coroutine_threadsafe to raise as if the loop were + # already closed. Patched via the dispatcher module's namespace + # so the dispatcher sees the fake at its call site. + def _raise_runtime(*_args: object, **_kwargs: object) -> object: + raise RuntimeError("Event loop is closed") + + monkeypatch.setattr( + f"{_DISPATCHER_MODULE}.asyncio.run_coroutine_threadsafe", + _raise_runtime, + ) + + with caplog.at_level(logging.DEBUG, logger=_DISPATCHER_LOGGER): + # Fire more calls than max_inflight — if the semaphore is + # NOT released on the RuntimeError path, we would see a + # "pool saturated" warning after 2 calls instead of the + # expected "loop unavailable" debug on every call. + for i in range(5): + dispatcher.dispatch(event_name=f"event.oops.{i}") + + # Provider never invoked — the loop is (simulated) dead. + assert provider.track_event_async.await_count == 0 + assert provider.track_event_async.call_count == 0 + + # Debug log fired for each attempt, and no saturation warning. + debug_hits = [ + r for r in caplog.records if "Telemetry loop unavailable" in r.message + ] + saturation_hits = [ + r for r in caplog.records if "Telemetry pool saturated" in r.message + ] + assert len(debug_hits) == 5, ( + f"expected 5 loop-unavailable debug logs, got {len(debug_hits)}" + ) + assert not saturation_hits, ( + "semaphore was not released on RuntimeError — pool saturated after 2 calls" + ) + finally: + dispatcher.shutdown() + + +def test_shutdown_swallows_when_loop_already_stopped(provider: MagicMock) -> None: + """Covers the ``RuntimeError`` catch in :meth:`shutdown`. + + If the loop is already stopped and closed by the time ``shutdown`` + is called (e.g. the ``_run_loop`` finally block has completed after + a direct external stop), ``call_soon_threadsafe`` raises. shutdown + must swallow and complete cleanly. + """ + dispatcher = LiveTrackEventDispatcher(provider, max_inflight=2) + + # Stop the loop directly, bypassing shutdown(). The ``_run_loop`` + # finally block will run: cancel tasks, gather, then close the + # loop. When our shutdown() then tries call_soon_threadsafe, the + # loop is already closed → RuntimeError. + dispatcher._loop.call_soon_threadsafe(dispatcher._loop.stop) + dispatcher._loop_thread.join(timeout=5.0) + assert not dispatcher._loop_thread.is_alive() + + # Must not raise; the internal RuntimeError from the already-closed + # loop is swallowed. + dispatcher.shutdown() + + # And is still idempotent afterwards. + dispatcher.shutdown() + + +def test_worker_exception_is_logged_at_debug( + provider: MagicMock, + dispatcher: LiveTrackEventDispatcher, + caplog: pytest.LogCaptureFixture, +) -> None: + """Covers the ``except Exception`` branch inside ``_run``. + + Complements :func:`test_worker_exception_does_not_propagate` by + asserting the log record (with ``exc_info``) actually fires, so + coverage records the debug-log line inside the coroutine. + """ + provider.track_event_async.side_effect = ValueError("bad payload") + + with caplog.at_level(logging.DEBUG, logger=_DISPATCHER_LOGGER): + dispatcher.dispatch(event_name="agent.bad") + # Wait for the coroutine to run AND the callback to fire so + # coverage collects the except-branch lines from the loop thread. + assert _wait_for(lambda: provider.track_event_async.await_count >= 1) + # Small sleep to let the callback finalize (release semaphore, + # drop from set) — otherwise coverage may race the callback. + time.sleep(0.05) + + matching = [ + r + for r in caplog.records + if "Failed to dispatch track_event" in r.message and r.levelno == logging.DEBUG + ] + assert matching, "expected a debug log for the swallowed exception" + # exc_info is attached so operators can trace the failure. + assert matching[0].exc_info is not None + assert isinstance(matching[0].exc_info[1], ValueError) + + +# --------------------------------------------------------------------------- +# thread safety +# --------------------------------------------------------------------------- + + +def test_dispatch_is_safe_from_many_threads(provider: MagicMock) -> None: + """dispatches from many caller threads all reach the provider. + + Exercises the semaphore, the futures set, and + ``run_coroutine_threadsafe`` under concurrent access from + non-loop threads. If the futures-set mutation weren't locked, this + would race and drop or duplicate futures under load. + """ + dispatcher = LiveTrackEventDispatcher(provider, max_inflight=128) + try: + n_threads = 50 + barrier = threading.Barrier(n_threads) + + def _fire(name: str) -> None: + # Wait until all threads are ready, then fire together. + barrier.wait(timeout=2.0) + dispatcher.dispatch(event_name=name) + + threads = [ + threading.Thread(target=_fire, args=(f"burst.{i}",)) + for i in range(n_threads) + ] + for t in threads: + t.start() + for t in threads: + t.join(timeout=5.0) + + assert _wait_for(lambda: provider.track_event_async.await_count == n_threads), ( + f"expected {n_threads} awaits, got {provider.track_event_async.await_count}" + ) + + seen = { + c.kwargs["event_name"] for c in provider.track_event_async.await_args_list + } + assert seen == {f"burst.{i}" for i in range(n_threads)} + finally: + dispatcher.shutdown() + + +# --------------------------------------------------------------------------- +# shutdown timeout +# --------------------------------------------------------------------------- + + +def test_shutdown_respects_timeout_when_drain_stalls(provider: MagicMock) -> None: + """``shutdown(wait=True, timeout=…)`` must return within the window + even if pending coroutines are stuck. + + Ensures a stalled backend cannot hang process teardown. Coroutines + still in flight past the timeout are cancelled by the loop's + teardown path. + """ + release = threading.Event() + + async def _never_finish(**_: object) -> None: + while not release.is_set(): + await asyncio.sleep(0.02) + + provider.track_event_async.side_effect = _never_finish + dispatcher = LiveTrackEventDispatcher(provider, max_inflight=4) + + dispatcher.dispatch(event_name="stuck.1") + dispatcher.dispatch(event_name="stuck.2") + assert _wait_for(lambda: provider.track_event_async.await_count >= 1) + + t0 = time.monotonic() + try: + dispatcher.shutdown(wait=True, timeout=0.1) + finally: + release.set() + elapsed = time.monotonic() - t0 + + # shutdown(timeout=0.1) waits ≤0.1s on the futures, then stops the + # loop and joins the thread (5s cap). Total must be well under a + # few seconds — a hang here would freeze the whole test suite. + assert elapsed < 3.0, f"shutdown took {elapsed:.3f}s with timeout=0.1s" + assert not dispatcher._loop_thread.is_alive() diff --git a/packages/uipath-platform/tests/services/test_llm_as_judge_validator.py b/packages/uipath-platform/tests/services/test_llm_as_judge_validator.py new file mode 100644 index 000000000..aba269511 --- /dev/null +++ b/packages/uipath-platform/tests/services/test_llm_as_judge_validator.py @@ -0,0 +1,142 @@ +"""Tests for LLMAsJudgeValidator — the decorator-path llm_as_judge validator. + +Verifies guardrail construction (validator_type + parameters), the +examples-only-when-present behavior, selector-is-None convention, stage support, +and input validation. Pure construction — no network. +""" + +from __future__ import annotations + +import pytest + +from uipath.platform.guardrails.decorators import ( + GuardrailExecutionStage, + LLMAsJudgeValidator, +) + +_RULE = "The answer must be genuinely funny, clean, and on-topic." + + +class TestLLMAsJudgeValidator: + """Tests for LLMAsJudgeValidator.""" + + def test_builds_guardrail_type(self): + guardrail = LLMAsJudgeValidator( + guardrail_text=_RULE, model="gpt-4o-2024-08-06" + ).get_built_in_guardrail(name="Judge", description=None, enabled_for_evals=True) + assert guardrail.guardrail_type == "builtInValidator" + assert guardrail.validator_type == "llm_as_judge" + + def test_required_parameters(self): + guardrail = LLMAsJudgeValidator( + guardrail_text=_RULE, model="gpt-4o-2024-08-06", threshold=2.0 + ).get_built_in_guardrail("Judge", None, True) + params = {p.id: p for p in guardrail.validator_parameters} + assert params["guardrailText"].parameter_type == "text" + assert params["guardrailText"].value == _RULE + assert params["model"].parameter_type == "enum" + assert params["model"].value == "gpt-4o-2024-08-06" + assert params["threshold"].parameter_type == "number" + assert params["threshold"].value == 2.0 + + def test_examples_only_when_present(self): + ids = { + p.id + for p in LLMAsJudgeValidator(guardrail_text=_RULE, model="m") + .get_built_in_guardrail("Judge", None, True) + .validator_parameters + } + assert "positiveExamples" not in ids + assert "negativeExamples" not in ids + + params = { + p.id: p + for p in LLMAsJudgeValidator( + guardrail_text=_RULE, + model="m", + positive_examples=["a clean pun"], + negative_examples=["not a joke"], + ) + .get_built_in_guardrail("Judge", None, True) + .validator_parameters + } + assert params["positiveExamples"].parameter_type == "text-list" + assert params["positiveExamples"].value == ["a clean pun"] + assert params["negativeExamples"].value == ["not a joke"] + + def test_selector_is_none(self): + # Decorator-path convention (matches PIIValidator / PromptInjectionValidator): + # scope comes from the decorated target, not the validator. + guardrail = LLMAsJudgeValidator( + guardrail_text=_RULE, model="m" + ).get_built_in_guardrail("Judge", None, True) + assert guardrail.selector is None + + def test_default_description(self): + guardrail = LLMAsJudgeValidator( + guardrail_text=_RULE, model="m" + ).get_built_in_guardrail("Judge", None, True) + assert guardrail.description == "LLM-as-judge evaluation" + + def test_all_stages_supported(self): + validator = LLMAsJudgeValidator(guardrail_text=_RULE, model="m") + assert validator.supported_stages == [] + # No stage restriction: both PRE and POST validate without raising. + validator.validate_stage(GuardrailExecutionStage.PRE) + validator.validate_stage(GuardrailExecutionStage.POST) + + def test_empty_guardrail_text_raises(self): + with pytest.raises(ValueError, match="guardrail_text"): + LLMAsJudgeValidator(guardrail_text=" ", model="m") + + def test_empty_model_raises(self): + with pytest.raises(ValueError, match="model"): + LLMAsJudgeValidator(guardrail_text=_RULE, model="") + + def test_threshold_out_of_range_raises(self): + with pytest.raises(ValueError, match="threshold"): + LLMAsJudgeValidator(guardrail_text=_RULE, model="m", threshold=7.0) + + def test_guardrail_text_over_limit_raises(self): + with pytest.raises(ValueError, match="guardrail_text exceeds"): + LLMAsJudgeValidator(guardrail_text="x" * 4001, model="m") + + def test_guardrail_text_at_limit_ok(self): + LLMAsJudgeValidator(guardrail_text="x" * 4000, model="m") + + def test_too_many_positive_examples_raises(self): + with pytest.raises(ValueError, match="positive_examples allows at most 2"): + LLMAsJudgeValidator( + guardrail_text=_RULE, model="m", positive_examples=["a", "b", "c"] + ) + + def test_too_many_negative_examples_raises(self): + with pytest.raises(ValueError, match="negative_examples allows at most 2"): + LLMAsJudgeValidator( + guardrail_text=_RULE, model="m", negative_examples=["a", "b", "c"] + ) + + def test_two_examples_each_ok(self): + LLMAsJudgeValidator( + guardrail_text=_RULE, + model="m", + positive_examples=["a", "b"], + negative_examples=["c", "d"], + ) + + def test_positive_example_over_length_raises(self): + with pytest.raises(ValueError, match="positive_examples entry"): + LLMAsJudgeValidator( + guardrail_text=_RULE, model="m", positive_examples=["x" * 1001] + ) + + def test_negative_example_over_length_raises(self): + with pytest.raises(ValueError, match="negative_examples entry"): + LLMAsJudgeValidator( + guardrail_text=_RULE, model="m", negative_examples=["x" * 1001] + ) + + def test_example_at_length_limit_ok(self): + LLMAsJudgeValidator( + guardrail_text=_RULE, model="m", positive_examples=["x" * 1000] + ) diff --git a/packages/uipath-platform/tests/services/test_llm_service.py b/packages/uipath-platform/tests/services/test_llm_service.py index e1ab8299b..4a6da2789 100644 --- a/packages/uipath-platform/tests/services/test_llm_service.py +++ b/packages/uipath-platform/tests/services/test_llm_service.py @@ -384,6 +384,104 @@ class Article(BaseModel): assert article_instance.title is None +class TestOpenAIServiceReasoningModelFiltering: + """Test that UiPathOpenAIService correctly filters out temperature for reasoning models.""" + + @pytest.fixture + def config(self): + return UiPathApiConfig(base_url="https://example.com", secret="test_secret") + + @pytest.fixture + def execution_context(self): + return UiPathExecutionContext() + + @pytest.fixture + def openai_service(self, config, execution_context): + return UiPathOpenAIService(config=config, execution_context=execution_context) + + @pytest.mark.parametrize( + "model", + [ + "o3-mini-2025-01-31", + "o4-mini-2025-04-16", + "o1-2024-12-17", + "o1-mini-2024-09-12", + "o3-2025-04-16", + ], + ) + @patch.object(UiPathOpenAIService, "request_async") + @pytest.mark.asyncio + async def test_reasoning_model_excludes_temperature( + self, mock_request, openai_service, model + ): + """Test that reasoning models do not include temperature in the request body.""" + mock_response = MagicMock() + mock_response.json.return_value = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1234567890, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + mock_request.return_value = mock_response + + await openai_service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model=model, + max_tokens=1000, + ) + + call_kwargs = mock_request.call_args[1] + request_body = call_kwargs["json"] + + assert "temperature" not in request_body, ( + f"Reasoning model {model} request must not include 'temperature'" + ) + assert request_body["max_tokens"] == 1000 + + @patch.object(UiPathOpenAIService, "request_async") + @pytest.mark.asyncio + async def test_non_reasoning_model_includes_temperature( + self, mock_request, openai_service + ): + """Test that non-reasoning models still include temperature.""" + mock_response = MagicMock() + mock_response.json.return_value = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1234567890, + "model": "gpt-4.1-mini-2025-04-14", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + mock_request.return_value = mock_response + + await openai_service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model="gpt-4.1-mini-2025-04-14", + max_tokens=1000, + temperature=0.7, + ) + + call_kwargs = mock_request.call_args[1] + request_body = call_kwargs["json"] + + assert request_body["temperature"] == 0.7 + + class TestNormalizedLlmServiceClaudeFiltering: """Test that Claude models correctly filter out OpenAI-specific parameters. @@ -543,3 +641,128 @@ async def test_claude_sonnet_45_excluded_params(self, mock_request, llm_service) assert "presence_penalty" not in request_body assert "top_p" not in request_body assert request_body["max_tokens"] == 8000 + + +class TestNormalizedLlmServiceReasoningModelFiltering: + """Test that reasoning models (o1, o3, o4) correctly filter out unsupported sampling parameters. + + OpenAI reasoning models do NOT support temperature, top_p, frequency_penalty, + presence_penalty, or n, and sending them causes 400 errors. + """ + + @pytest.fixture + def config(self): + return UiPathApiConfig(base_url="https://example.com", secret="test_secret") + + @pytest.fixture + def execution_context(self): + return UiPathExecutionContext() + + @pytest.fixture + def llm_service(self, config, execution_context): + from uipath.platform.chat._llm_gateway_service import UiPathLlmChatService + + return UiPathLlmChatService(config=config, execution_context=execution_context) + + @pytest.mark.parametrize( + "model", + [ + "o3-mini-2025-01-31", + "o4-mini-2025-04-16", + "o1-2024-12-17", + "o1-mini-2024-09-12", + "o3-2025-04-16", + ], + ) + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathLlmChatService.request_async" + ) + @pytest.mark.asyncio + async def test_reasoning_model_excludes_sampling_params( + self, mock_request, llm_service, model + ): + """Test that reasoning models do not include temperature or other sampling params.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1234567890, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + mock_request.return_value = mock_response + + await llm_service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model=model, + max_tokens=1000, + ) + + call_kwargs = mock_request.call_args[1] + request_body = call_kwargs["json"] + + assert "temperature" not in request_body, ( + f"Reasoning model {model} request must not include 'temperature'" + ) + assert "n" not in request_body, ( + f"Reasoning model {model} request must not include 'n'" + ) + assert "frequency_penalty" not in request_body, ( + f"Reasoning model {model} request must not include 'frequency_penalty'" + ) + assert "presence_penalty" not in request_body, ( + f"Reasoning model {model} request must not include 'presence_penalty'" + ) + assert "top_p" not in request_body, ( + f"Reasoning model {model} request must not include 'top_p'" + ) + assert request_body["max_tokens"] == 1000 + + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathLlmChatService.request_async" + ) + @pytest.mark.asyncio + async def test_non_reasoning_model_includes_temperature( + self, mock_request, llm_service + ): + """Test that non-reasoning models still include temperature and sampling params.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1234567890, + "model": "gpt-4.1-mini-2025-04-14", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + mock_request.return_value = mock_response + + await llm_service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model="gpt-4.1-mini-2025-04-14", + max_tokens=1000, + temperature=0.5, + ) + + call_kwargs = mock_request.call_args[1] + request_body = call_kwargs["json"] + + assert request_body["temperature"] == 0.5 + assert "n" in request_body + assert "frequency_penalty" in request_body + assert "presence_penalty" in request_body diff --git a/packages/uipath-platform/tests/services/test_llm_temperature_skip.py b/packages/uipath-platform/tests/services/test_llm_temperature_skip.py new file mode 100644 index 000000000..fd3aef196 --- /dev/null +++ b/packages/uipath-platform/tests/services/test_llm_temperature_skip.py @@ -0,0 +1,245 @@ +"""Temperature is dropped for models that reject it, per LLM Gateway discovery. + +Regression coverage for Sonnet 5 evals: the judge sent `temperature` on every +normalized chat completion, and the gateway answered 400 "`temperature` is +deprecated for this model." +""" + +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest + +from uipath.platform import UiPathApiConfig, UiPathExecutionContext +from uipath.platform.chat import UiPathLlmChatService, UiPathOpenAIService +from uipath.platform.chat._model_capabilities import _reset_cache + +SKIP_TEMPERATURE_MODEL = "anthropic.claude-sonnet-5" + +SKIP_TEMPERATURE_DISCOVERY = [ + { + "modelName": SKIP_TEMPERATURE_MODEL, + "modelDetails": {"shouldSkipTemperature": True}, + } +] + + +def _discovery_response(models: list[dict[str, Any]]) -> MagicMock: + response = MagicMock() + response.status_code = 200 + response.json.return_value = models + return response + + +def _completion_response(model: str) -> MagicMock: + response = MagicMock() + response.status_code = 200 + response.json.return_value = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1234567890, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + return response + + +def _responder(discovery: list[dict[str, Any]] | Exception, model: str = "test-model"): + """Answer by endpoint, so assertions don't depend on call ordering.""" + + async def respond(_method, endpoint, **_kwargs): + if "discovery" in str(endpoint): + if isinstance(discovery, Exception): + raise discovery + return _discovery_response(discovery) + return _completion_response(model) + + return respond + + +def _is_discovery(call) -> bool: + return "discovery" in str(call.args[1]) + + +def _discovery_calls(mock_request: MagicMock) -> list[Any]: + return [c for c in mock_request.call_args_list if _is_discovery(c)] + + +def _completion_body(mock_request: MagicMock) -> dict[str, Any]: + completions = [c for c in mock_request.call_args_list if not _is_discovery(c)] + assert completions, "no chat completion request was sent" + return completions[-1][1]["json"] + + +class TestSkipTemperatureFromDiscovery: + @pytest.fixture(autouse=True) + def clear_discovery_cache(self): + # The cache is process-wide; keep it from leaking in either direction. + _reset_cache() + yield + _reset_cache() + + @pytest.fixture + def config(self): + return UiPathApiConfig(base_url="https://example.com", secret="test_secret") + + @pytest.fixture + def execution_context(self): + return UiPathExecutionContext() + + @pytest.fixture + def llm_service(self, config, execution_context): + return UiPathLlmChatService(config=config, execution_context=execution_context) + + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathLlmChatService.request_async" + ) + @pytest.mark.asyncio + async def test_omits_temperature_when_discovery_sets_flag( + self, mock_request, llm_service + ): + mock_request.side_effect = _responder( + SKIP_TEMPERATURE_DISCOVERY, SKIP_TEMPERATURE_MODEL + ) + + await llm_service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model=SKIP_TEMPERATURE_MODEL, + temperature=0, + ) + + assert "temperature" not in _completion_body(mock_request) + + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathLlmChatService.request_async" + ) + @pytest.mark.asyncio + async def test_keeps_temperature_when_flag_absent(self, mock_request, llm_service): + mock_request.side_effect = _responder( + [ + { + "modelName": "gpt-4o-2024-11-20", + "modelDetails": {"maxOutputTokens": 4096}, + } + ], + "gpt-4o-2024-11-20", + ) + + await llm_service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model="gpt-4o-2024-11-20", + temperature=0.3, + ) + + assert _completion_body(mock_request)["temperature"] == 0.3 + + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathLlmChatService.request_async" + ) + @pytest.mark.asyncio + async def test_keeps_temperature_for_model_missing_from_discovery( + self, mock_request, llm_service + ): + mock_request.side_effect = _responder( + SKIP_TEMPERATURE_DISCOVERY, "some-other-model" + ) + + await llm_service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model="some-other-model", + temperature=0, + ) + + assert _completion_body(mock_request)["temperature"] == 0 + + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathLlmChatService.request_async" + ) + @pytest.mark.asyncio + async def test_keeps_temperature_when_discovery_unavailable( + self, mock_request, llm_service + ): + """Discovery being down must not fail or silently alter the LLM call.""" + mock_request.side_effect = _responder( + RuntimeError("discovery unreachable"), SKIP_TEMPERATURE_MODEL + ) + + await llm_service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model=SKIP_TEMPERATURE_MODEL, + temperature=0, + ) + + assert _completion_body(mock_request)["temperature"] == 0 + + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathLlmChatService.request_async" + ) + @pytest.mark.asyncio + async def test_discovery_fetched_once_for_repeated_calls( + self, mock_request, llm_service + ): + mock_request.side_effect = _responder( + SKIP_TEMPERATURE_DISCOVERY, SKIP_TEMPERATURE_MODEL + ) + + for _ in range(2): + await llm_service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model=SKIP_TEMPERATURE_MODEL, + temperature=0, + ) + + assert len(_discovery_calls(mock_request)) == 1 + assert "temperature" not in _completion_body(mock_request) + + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathLlmChatService.request_async" + ) + @pytest.mark.asyncio + async def test_discovery_scoped_by_agenthub_config( + self, mock_request, config, execution_context + ): + service = UiPathLlmChatService( + config=config, + execution_context=execution_context, + agenthub_config="agentsevals", + ) + mock_request.side_effect = _responder([], SKIP_TEMPERATURE_MODEL) + + await service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model=SKIP_TEMPERATURE_MODEL, + ) + + discovery_headers = _discovery_calls(mock_request)[0][1]["headers"] + assert discovery_headers["x-uipath-agenthub-config"] == "agentsevals" + + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathOpenAIService.request_async" + ) + @pytest.mark.asyncio + async def test_openai_compatible_path_omits_temperature( + self, mock_request, config, execution_context + ): + service = UiPathOpenAIService( + config=config, execution_context=execution_context + ) + mock_request.side_effect = _responder( + SKIP_TEMPERATURE_DISCOVERY, SKIP_TEMPERATURE_MODEL + ) + + await service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model=SKIP_TEMPERATURE_MODEL, + temperature=0, + ) + + assert "temperature" not in _completion_body(mock_request) diff --git a/packages/uipath-platform/tests/services/test_llm_trace_context.py b/packages/uipath-platform/tests/services/test_llm_trace_context.py new file mode 100644 index 000000000..a05032113 --- /dev/null +++ b/packages/uipath-platform/tests/services/test_llm_trace_context.py @@ -0,0 +1,227 @@ +"""Tests for build_trace_context_headers.""" + +import os +from unittest.mock import patch + +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.trace import NonRecordingSpan, SpanContext, TraceFlags +from uipath.core.feature_flags import FeatureFlags + +from uipath.platform.chat.llm_trace_context import build_trace_context_headers +from uipath.platform.constants import ENV_PROJECT_KEY + +FEATURE_FLAG = "EnableTraceContextHeaders" + + +def _make_span(): + """Create a real OTEL span for testing.""" + provider = TracerProvider() + tracer = provider.get_tracer("test") + return tracer.start_span("test-span") + + +class TestFeatureFlagDisabled: + """When the feature flag is off, no headers are returned.""" + + def setup_method(self) -> None: + FeatureFlags.reset_flags() + + def test_returns_empty_dict_by_default(self) -> None: + assert build_trace_context_headers() == {} + + def test_returns_empty_dict_when_explicitly_disabled(self) -> None: + FeatureFlags.configure_flags({FEATURE_FLAG: False}) + assert build_trace_context_headers() == {} + + +class TestTraceparentHeader: + """When enabled, x-uipath-traceparent-id is populated from config + span.""" + + def setup_method(self) -> None: + FeatureFlags.reset_flags() + FeatureFlags.configure_flags({FEATURE_FLAG: True}) + + def test_traceparent_from_config_and_span(self) -> None: + span = _make_span() + ctx = span.get_span_context() + # OTEL span id is 64-bit => 16 hex chars; traceparent carries the trailing flags + # segment. The gateway's strict W3C parser requires exactly this shape. + expected_span_id = format(ctx.span_id, "016x") + config_trace = "abcdef1234567890abcdef1234567890" + env = {"UIPATH_TRACE_ID": config_trace} + with ( + patch.dict(os.environ, env), + patch( + "uipath.platform.chat.llm_trace_context.trace.get_current_span", + return_value=span, + ), + ): + headers = build_trace_context_headers() + + assert "x-uipath-traceparent-id" in headers + value = headers["x-uipath-traceparent-id"] + assert value == f"00-{config_trace}-{expected_span_id}-01" + parts = value.split("-") + assert len(parts) == 4 + assert parts[0] == "00" + assert len(parts[1]) == 32 + assert len(parts[2]) == 16 + assert parts[3] == "01" + + def test_no_traceparent_without_config_trace_id(self) -> None: + headers = build_trace_context_headers() + assert "x-uipath-traceparent-id" not in headers + + def test_traceparent_strips_dashes_from_config_trace_id(self) -> None: + span = _make_span() + uuid_trace = "abcdef12-3456-7890-abcd-ef1234567890" + env = {"UIPATH_TRACE_ID": uuid_trace} + with ( + patch.dict(os.environ, env), + patch( + "uipath.platform.chat.llm_trace_context.trace.get_current_span", + return_value=span, + ), + ): + headers = build_trace_context_headers() + + value = headers["x-uipath-traceparent-id"] + parts = value.split("-") + assert parts[1] == "abcdef1234567890abcdef1234567890" + + def test_no_traceparent_with_invalid_span(self) -> None: + ctx = SpanContext( + trace_id=0, + span_id=0, + is_remote=False, + trace_flags=TraceFlags(0), + ) + span = NonRecordingSpan(ctx) + env = {"UIPATH_TRACE_ID": "abcdef1234567890abcdef1234567890"} + with ( + patch.dict(os.environ, env), + patch( + "uipath.platform.chat.llm_trace_context.trace.get_current_span", + return_value=span, + ), + ): + headers = build_trace_context_headers() + + assert "x-uipath-traceparent-id" not in headers + + +class TestBaggageHeader: + """When enabled, x-uipath-tracebaggage is populated from UiPathConfig.""" + + def setup_method(self) -> None: + from uipath.platform.common._span_utils import _read_config_id + + _read_config_id.cache_clear() + FeatureFlags.reset_flags() + FeatureFlags.configure_flags({FEATURE_FLAG: True}) + + def test_all_env_vars_present(self) -> None: + env = { + "UIPATH_FOLDER_KEY": "folder-abc", + ENV_PROJECT_KEY: "agent-123", + "UIPATH_PROCESS_UUID": "process-789", + "UIPATH_JOB_KEY": "job-456", + } + with patch.dict(os.environ, env, clear=True): + headers = build_trace_context_headers() + + baggage = headers["x-uipath-tracebaggage"] + assert "folderKey=folder-abc" in baggage + assert "agentId=agent-123" in baggage + assert "processKey=process-789" in baggage + assert "jobKey=job-456" in baggage + + def test_partial_env_vars(self) -> None: + env = {"UIPATH_FOLDER_KEY": "folder-only"} + with patch.dict(os.environ, env, clear=True): + headers = build_trace_context_headers() + + baggage = headers["x-uipath-tracebaggage"] + assert "folderKey=folder-only" in baggage + + def test_agent_id_from_project_key_env(self) -> None: + env = {ENV_PROJECT_KEY: "real-agent-id"} + with patch.dict(os.environ, env, clear=True): + headers = build_trace_context_headers() + + baggage = headers["x-uipath-tracebaggage"] + assert "agentId=real-agent-id" in baggage + + def test_no_agent_id_without_env_vars(self) -> None: + env = {"UIPATH_FOLDER_KEY": "f1"} + with patch.dict(os.environ, env, clear=True): + headers = build_trace_context_headers() + + baggage = headers["x-uipath-tracebaggage"] + assert "agentId" not in baggage + assert "folderKey=f1" in baggage + + def test_no_baggage_without_env_vars(self) -> None: + with patch.dict(os.environ, {}, clear=True): + headers = build_trace_context_headers() + + assert "x-uipath-tracebaggage" not in headers + + def test_baggage_comma_separated(self) -> None: + env = { + "UIPATH_FOLDER_KEY": "f1", + ENV_PROJECT_KEY: "a1", + } + with patch.dict(os.environ, env, clear=True): + headers = build_trace_context_headers() + + baggage = headers["x-uipath-tracebaggage"] + parts = baggage.split(",") + assert len(parts) == 2 # folderKey + agentId + + def test_extra_baggage_included(self) -> None: + env = {"UIPATH_FOLDER_KEY": "f1"} + with patch.dict(os.environ, env, clear=True): + headers = build_trace_context_headers(extra_baggage=["source=agents"]) + + baggage = headers["x-uipath-tracebaggage"] + assert "source=agents" in baggage + assert "folderKey=f1" in baggage + + def test_extra_baggage_only(self) -> None: + with patch.dict(os.environ, {}, clear=True): + headers = build_trace_context_headers( + extra_baggage=["source=agents", "custom=value"] + ) + + baggage = headers["x-uipath-tracebaggage"] + assert baggage == "source=agents,custom=value" + + +class TestBothHeaders: + """When enabled with an active span and env vars, both headers are present.""" + + def setup_method(self) -> None: + FeatureFlags.reset_flags() + FeatureFlags.configure_flags({FEATURE_FLAG: True}) + + def test_both_headers_present(self) -> None: + span = _make_span() + env = { + "UIPATH_FOLDER_KEY": "folder-abc", + "UIPATH_TRACE_ID": "abcdef1234567890abcdef1234567890", + } + with ( + patch.dict(os.environ, env, clear=True), + patch( + "uipath.platform.chat.llm_trace_context.trace.get_current_span", + return_value=span, + ), + ): + headers = build_trace_context_headers() + + assert "x-uipath-traceparent-id" in headers + assert headers["x-uipath-traceparent-id"].startswith( + "00-abcdef1234567890abcdef1234567890-" + ) + assert "x-uipath-tracebaggage" in headers diff --git a/packages/uipath-platform/tests/services/test_mcp_service.py b/packages/uipath-platform/tests/services/test_mcp_service.py index fdc5d8ee1..002eb981f 100644 --- a/packages/uipath-platform/tests/services/test_mcp_service.py +++ b/packages/uipath-platform/tests/services/test_mcp_service.py @@ -1,10 +1,14 @@ -from unittest.mock import Mock, patch +from unittest.mock import AsyncMock, Mock, patch import pytest from pytest_httpx import HTTPXMock from uipath.platform import UiPathApiConfig, UiPathExecutionContext -from uipath.platform.common.constants import HEADER_FOLDER_KEY, HEADER_USER_AGENT +from uipath.platform.common._bindings import ( + GenericResourceOverwrite, + _resource_overwrites, +) +from uipath.platform.constants import HEADER_FOLDER_KEY, HEADER_USER_AGENT from uipath.platform.orchestrator import McpService from uipath.platform.orchestrator._folder_service import FolderService from uipath.platform.orchestrator.mcp import McpServer @@ -363,6 +367,125 @@ async def test_retrieve_server_async( == f"UiPath.Python.Sdk/UiPath.Python.Sdk.Activities.McpService.retrieve_async/{version}" ) + def test_retrieve_server_by_name(self, service: McpService) -> None: + response = Mock() + response.json.return_value = { + "name": "Friendly MCP/Europe", + "slug": "friendly-mcp-europe", + } + + with patch.object(service, "request", return_value=response) as request: + server = service.retrieve(name="Friendly MCP/Europe") + + assert server.name == "Friendly MCP/Europe" + assert "api/servers/Friendly%20MCP%2FEurope" in str( + request.call_args.kwargs["url"] + ) + + def test_retrieve_rejects_name_and_slug(self, service: McpService) -> None: + with pytest.raises( + ValueError, match="Specify either 'name' or 'slug', not both" + ): + service.retrieve("friendly-mcp", name="Friendly MCP") + + def test_retrieve_requires_name_or_slug(self, service: McpService) -> None: + with pytest.raises( + TypeError, match="Either 'name' or 'slug' must be provided" + ): + service.retrieve() + + def test_retrieve_applies_display_name_binding( + self, service: McpService + ) -> None: + response = Mock() + response.json.return_value = { + "name": "Replacement MCP", + "slug": "replacement-mcp", + } + overwrite = GenericResourceOverwrite( + resource_type="mcpServer", + name="Replacement MCP", + folder_path="Replacement Folder", + ) + token = _resource_overwrites.set({"mcpServer.Original MCP": overwrite}) + + try: + with ( + patch.object(service, "request", return_value=response) as request, + patch.object( + service._folders_service, + "retrieve_folder_key", + return_value="replacement-folder-key", + ), + ): + service.retrieve(name="Original MCP") + finally: + _resource_overwrites.reset(token) + + assert "api/servers/Replacement%20MCP" in str( + request.call_args.kwargs["url"] + ) + assert ( + request.call_args.kwargs["headers"][HEADER_FOLDER_KEY] + == "replacement-folder-key" + ) + + def test_retrieve_applies_legacy_slug_binding( + self, service: McpService + ) -> None: + response = Mock() + response.json.return_value = { + "name": "Replacement MCP", + "slug": "replacement-mcp", + } + overwrite = GenericResourceOverwrite( + resource_type="mcpServer", + name="Replacement MCP", + folder_path="Replacement Folder", + ) + token = _resource_overwrites.set({"mcpServer.original-mcp": overwrite}) + + try: + with ( + patch.object(service, "request", return_value=response) as request, + patch.object( + service._folders_service, + "retrieve_folder_key", + return_value="replacement-folder-key", + ), + ): + service.retrieve(slug="original-mcp") + finally: + _resource_overwrites.reset(token) + + assert "api/servers/Replacement%20MCP" in str( + request.call_args.kwargs["url"] + ) + assert ( + request.call_args.kwargs["headers"][HEADER_FOLDER_KEY] + == "replacement-folder-key" + ) + + @pytest.mark.anyio + async def test_retrieve_server_by_name_async(self, service: McpService) -> None: + response = Mock() + response.json.return_value = { + "name": "Friendly MCP/Europe", + "slug": "friendly-mcp-europe", + } + + with patch.object( + service, + "request_async", + new=AsyncMock(return_value=response), + ) as request: + server = await service.retrieve_async(name="Friendly MCP/Europe") + + assert server.name == "Friendly MCP/Europe" + assert "api/servers/Friendly%20MCP%2FEurope" in str( + request.call_args.kwargs["url"] + ) + class TestRequestKwargs: """Test that all methods pass the correct kwargs to request/request_async.""" @@ -551,3 +674,27 @@ async def test_retrieve_async_passes_all_kwargs( call_kwargs.kwargs["headers"][HEADER_FOLDER_KEY] == "test-folder-key" ) + + +class TestMcpServerType: + """Tests for the McpServerType enum and McpServer validation.""" + + def test_swagger_type_value(self) -> None: + from uipath.platform.orchestrator.mcp import McpServerType + + assert McpServerType.Swagger == 7 + + def test_validate_swagger_server(self) -> None: + """A Swagger (type=7) server must validate — regression for backend + server types newer than the SDK's enum.""" + server = McpServer.model_validate( + {"slug": "contoso-directory", "name": "Employee Directory", "type": 7} + ) + assert server.type == 7 + assert server.slug == "contoso-directory" + + +def test_mcp_retrieve_spec_encodes_display_name(service: McpService) -> None: + spec = service._retrieve_spec(name="Friendly MCP/Europe", folder_path=None) + + assert "api/servers/Friendly%20MCP%2FEurope" in str(spec.endpoint) diff --git a/packages/uipath-platform/tests/services/test_memory_service.py b/packages/uipath-platform/tests/services/test_memory_service.py new file mode 100644 index 000000000..716e3438c --- /dev/null +++ b/packages/uipath-platform/tests/services/test_memory_service.py @@ -0,0 +1,504 @@ +"""Unit tests for MemoryService with HTTP mocking.""" + +import json + +import pytest +from pytest_httpx import HTTPXMock + +from uipath.platform import UiPathApiConfig, UiPathExecutionContext +from uipath.platform.memory import ( + EscalationMemoryIngestRequest, + EscalationMemorySearchResponse, + MemoryMatch, + MemoryMatchField, + MemorySearchRequest, + MemorySearchResponse, + MemorySpace, + MemorySpaceListResponse, + SearchField, + SearchMode, + SearchSettings, +) +from uipath.platform.memory._memory_service import MemoryService +from uipath.platform.orchestrator._folder_service import FolderService + + +@pytest.fixture +def folder_service( + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, +) -> FolderService: + return FolderService(config=config, execution_context=execution_context) + + +@pytest.fixture +def service( + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + folder_service: FolderService, + monkeypatch: pytest.MonkeyPatch, +) -> MemoryService: + monkeypatch.setenv("UIPATH_FOLDER_KEY", "test-folder-key") + return MemoryService( + config=config, + execution_context=execution_context, + folders_service=folder_service, + ) + + +# ── Sample response payloads ────────────────────────────────────────── + +SAMPLE_INDEX = { + "id": "aaaa-bbbb-cccc-dddd", + "name": "test-memory-space", + "description": "A test memory space", + "lastQueried": "2026-03-30T00:00:00Z", + "memoriesCount": 5, + "folderKey": "test-folder-key", + "createdByUserId": "user-123", + "isEncrypted": False, +} + +SAMPLE_LIST_RESPONSE = {"value": [SAMPLE_INDEX]} + +SAMPLE_SEARCH_RESPONSE = { + "results": [ + { + "memoryItemId": "item-001", + "score": 0.95, + "semanticScore": 0.92, + "weightedScore": 0.93, + "fields": [ + { + "keyPath": ["input"], + "value": "What is the capital of France?", + "weight": 1.0, + "score": 0.95, + "weightedScore": 0.95, + } + ], + "span": None, + "feedback": None, + } + ], + "metadata": {"queryTime": "12ms"}, + "systemPromptInjection": "Based on past interactions: Paris is the capital.", +} + +SAMPLE_ESCALATION_SEARCH_RESPONSE = { + "results": [ + { + "answer": { + "output": {"action": "approve", "reason": "meets criteria"}, + "outcome": "approved", + } + } + ], +} + + +class TestMemoryService: + """Unit tests for MemoryService.""" + + class TestCreate: + def test_create_memory_space( + self, + httpx_mock: HTTPXMock, + service: MemoryService, + base_url: str, + org: str, + tenant: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/episodicmemories/create", + status_code=200, + json=SAMPLE_INDEX, + ) + + result = service.create( + name="test-memory-space", + description="A test memory space", + ) + + assert isinstance(result, MemorySpace) + assert result.id == "aaaa-bbbb-cccc-dddd" + assert result.name == "test-memory-space" + assert result.memories_count == 5 + + sent = httpx_mock.get_request() + assert sent is not None + assert sent.method == "POST" + body = json.loads(sent.content) + assert body["name"] == "test-memory-space" + assert body["description"] == "A test memory space" + + def test_create_sends_folder_header( + self, + httpx_mock: HTTPXMock, + service: MemoryService, + base_url: str, + org: str, + tenant: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/episodicmemories/create", + status_code=200, + json=SAMPLE_INDEX, + ) + + service.create(name="test", folder_key="custom-folder-key") + + sent = httpx_mock.get_request() + assert sent is not None + assert sent.headers.get("x-uipath-folderkey") == "custom-folder-key" + + def test_create_with_encryption( + self, + httpx_mock: HTTPXMock, + service: MemoryService, + base_url: str, + org: str, + tenant: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/episodicmemories/create", + status_code=200, + json={**SAMPLE_INDEX, "isEncrypted": True}, + ) + + result = service.create( + name="encrypted-space", + is_encrypted=True, + ) + + assert result.is_encrypted is True + sent = httpx_mock.get_request() + assert sent is not None + body = json.loads(sent.content) + assert body["isEncrypted"] is True + + class TestList: + def test_list_memory_spaces( + self, + httpx_mock: HTTPXMock, + service: MemoryService, + base_url: str, + org: str, + tenant: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/episodicmemories", + status_code=200, + json=SAMPLE_LIST_RESPONSE, + ) + + result = service.list() + + assert isinstance(result, MemorySpaceListResponse) + assert len(result.value) == 1 + assert result.value[0].name == "test-memory-space" + + def test_list_with_odata_params( + self, + httpx_mock: HTTPXMock, + service: MemoryService, + base_url: str, + org: str, + tenant: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/episodicmemories?%24filter=Name+eq+%27test%27&%24orderby=Name+asc&%24top=10&%24skip=5", + status_code=200, + json=SAMPLE_LIST_RESPONSE, + ) + + result = service.list( + filter="Name eq 'test'", + orderby="Name asc", + top=10, + skip=5, + ) + + assert isinstance(result, MemorySpaceListResponse) + + def test_list_empty( + self, + httpx_mock: HTTPXMock, + service: MemoryService, + base_url: str, + org: str, + tenant: str, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/ecs_/v2/episodicmemories", + status_code=200, + json={"value": []}, + ) + + result = service.list() + + assert isinstance(result, MemorySpaceListResponse) + assert len(result.value) == 0 + + class TestSearch: + def test_search_memory( + self, + httpx_mock: HTTPXMock, + service: MemoryService, + base_url: str, + org: str, + tenant: str, + ) -> None: + memory_space_id = "aaaa-bbbb-cccc-dddd" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/llmopstenant_/api/Agent/memory/{memory_space_id}/search", + status_code=200, + json=SAMPLE_SEARCH_RESPONSE, + ) + + request = MemorySearchRequest( + fields=[ + SearchField( + key_path=["input"], + value="What is the capital of France?", + ) + ], + settings=SearchSettings( + threshold=0.0, + result_count=5, + search_mode=SearchMode.Hybrid, + ), + definition_system_prompt="You are a helpful assistant.", + ) + + result = service.search( + memory_space_id=memory_space_id, + request=request, + ) + + assert isinstance(result, MemorySearchResponse) + assert len(result.results) == 1 + assert isinstance(result.results[0], MemoryMatch) + assert result.results[0].memory_item_id == "item-001" + assert result.results[0].score == 0.95 + assert isinstance(result.results[0].fields[0], MemoryMatchField) + assert ( + result.system_prompt_injection + == "Based on past interactions: Paris is the capital." + ) + + sent = httpx_mock.get_request() + assert sent is not None + assert sent.method == "POST" + body = json.loads(sent.content) + assert body["fields"][0]["keyPath"] == ["input"] + assert body["settings"]["searchMode"] == "Hybrid" + assert body["definitionSystemPrompt"] == "You are a helpful assistant." + + def test_search_sends_folder_header( + self, + httpx_mock: HTTPXMock, + service: MemoryService, + base_url: str, + org: str, + tenant: str, + ) -> None: + memory_space_id = "aaaa-bbbb-cccc-dddd" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/llmopstenant_/api/Agent/memory/{memory_space_id}/search", + status_code=200, + json=SAMPLE_SEARCH_RESPONSE, + ) + + request = MemorySearchRequest( + fields=[SearchField(key_path=["input"], value="test")], + settings=SearchSettings( + threshold=0.0, + result_count=1, + search_mode=SearchMode.Semantic, + ), + ) + + service.search( + memory_space_id=memory_space_id, + request=request, + folder_key="custom-folder", + ) + + sent = httpx_mock.get_request() + assert sent is not None + assert sent.headers.get("x-uipath-folderkey") == "custom-folder" + + class TestEscalationSearch: + def test_escalation_search( + self, + httpx_mock: HTTPXMock, + service: MemoryService, + base_url: str, + org: str, + tenant: str, + ) -> None: + memory_space_id = "aaaa-bbbb-cccc-dddd" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/llmopstenant_/api/Agent/memory/{memory_space_id}/escalation/search", + status_code=200, + json=SAMPLE_ESCALATION_SEARCH_RESPONSE, + ) + + request = MemorySearchRequest( + fields=[SearchField(key_path=["input"], value="approval request")], + settings=SearchSettings( + threshold=0.0, + result_count=5, + search_mode=SearchMode.Hybrid, + ), + ) + + result = service.escalation_search( + memory_space_id=memory_space_id, + request=request, + ) + + assert isinstance(result, EscalationMemorySearchResponse) + assert result.results is not None + assert len(result.results) == 1 + assert result.results[0].answer is not None + assert result.results[0].answer.outcome == "approved" + + sent = httpx_mock.get_request() + assert sent is not None + assert sent.method == "POST" + assert "/escalation/search" in str(sent.url) + + def test_escalation_search_empty_results( + self, + httpx_mock: HTTPXMock, + service: MemoryService, + base_url: str, + org: str, + tenant: str, + ) -> None: + memory_space_id = "aaaa-bbbb-cccc-dddd" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/llmopstenant_/api/Agent/memory/{memory_space_id}/escalation/search", + status_code=200, + json={"results": None}, + ) + + request = MemorySearchRequest( + fields=[SearchField(key_path=["input"], value="no match")], + settings=SearchSettings( + threshold=0.0, + result_count=1, + search_mode=SearchMode.Hybrid, + ), + ) + + result = service.escalation_search( + memory_space_id=memory_space_id, + request=request, + ) + + assert isinstance(result, EscalationMemorySearchResponse) + assert result.results is None + + class TestEscalationIngest: + def test_escalation_ingest( + self, + httpx_mock: HTTPXMock, + service: MemoryService, + base_url: str, + org: str, + tenant: str, + ) -> None: + memory_space_id = "aaaa-bbbb-cccc-dddd" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/llmopstenant_/api/Agent/memory/{memory_space_id}/escalation/ingest", + status_code=200, + ) + + request = EscalationMemoryIngestRequest( + span_id="span-123", + trace_id="trace-456", + answer='{"action": "approve"}', + attributes='{"input": "approve this?"}', + user_id="user-789", + ) + + service.escalation_ingest( + memory_space_id=memory_space_id, + request=request, + ) + + sent = httpx_mock.get_request() + assert sent is not None + assert sent.method == "POST" + assert "/escalation/ingest" in str(sent.url) + body = json.loads(sent.content) + assert body["spanId"] == "span-123" + assert body["traceId"] == "trace-456" + assert body["answer"] == '{"action": "approve"}' + assert body["attributes"] == '{"input": "approve this?"}' + assert body["userId"] == "user-789" + + def test_escalation_ingest_sends_folder_header( + self, + httpx_mock: HTTPXMock, + service: MemoryService, + base_url: str, + org: str, + tenant: str, + ) -> None: + memory_space_id = "aaaa-bbbb-cccc-dddd" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/llmopstenant_/api/Agent/memory/{memory_space_id}/escalation/ingest", + status_code=200, + ) + + request = EscalationMemoryIngestRequest( + span_id="s1", + trace_id="t1", + answer="yes", + attributes="{}", + ) + + service.escalation_ingest( + memory_space_id=memory_space_id, + request=request, + folder_key="my-folder", + ) + + sent = httpx_mock.get_request() + assert sent is not None + assert sent.headers.get("x-uipath-folderkey") == "my-folder" + + def test_escalation_ingest_excludes_none_user_id( + self, + httpx_mock: HTTPXMock, + service: MemoryService, + base_url: str, + org: str, + tenant: str, + ) -> None: + memory_space_id = "aaaa-bbbb-cccc-dddd" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/llmopstenant_/api/Agent/memory/{memory_space_id}/escalation/ingest", + status_code=200, + ) + + request = EscalationMemoryIngestRequest( + span_id="s1", + trace_id="t1", + answer="yes", + attributes="{}", + ) + + service.escalation_ingest( + memory_space_id=memory_space_id, + request=request, + ) + + sent = httpx_mock.get_request() + assert sent is not None + body = json.loads(sent.content) + assert "userId" not in body diff --git a/packages/uipath-platform/tests/services/test_memory_service_e2e.py b/packages/uipath-platform/tests/services/test_memory_service_e2e.py new file mode 100644 index 000000000..6c7866611 --- /dev/null +++ b/packages/uipath-platform/tests/services/test_memory_service_e2e.py @@ -0,0 +1,164 @@ +"""E2E tests for MemoryService against real ECS + LLMOps endpoints. + +Prerequisites: + uipath auth --alpha # sets UIPATH_URL + UIPATH_ACCESS_TOKEN + export UIPATH_FOLDER_KEY=... # folder GUID with agent memory enabled + +Run: + cd packages/uipath-platform + uv run pytest tests/services/test_memory_service_e2e.py -m e2e -v +""" + +import os +import uuid + +import pytest + +from uipath.platform import UiPath +from uipath.platform.memory import ( + EscalationMemorySearchResponse, + MemorySearchRequest, + MemorySearchResponse, + MemorySpace, + MemorySpaceListResponse, + SearchField, + SearchMode, + SearchSettings, +) + +pytestmark = pytest.mark.e2e + + +def _require_env(name: str) -> str: + value = os.environ.get(name) + if not value: + pytest.skip(f"Environment variable {name} is not set") + return value + + +@pytest.fixture(scope="module") +def sdk() -> UiPath: + """Create a real UiPath client from env vars. + + Supports two auth modes: + - Token-based: UIPATH_URL + UIPATH_ACCESS_TOKEN (from `uipath auth`) + - Client credentials: UIPATH_URL + UIPATH_CLIENT_ID + UIPATH_CLIENT_SECRET (CI) + """ + _require_env("UIPATH_URL") + client_id = os.environ.get("UIPATH_CLIENT_ID") + client_secret = os.environ.get("UIPATH_CLIENT_SECRET") + if client_id and client_secret: + return UiPath(client_id=client_id, client_secret=client_secret) + _require_env("UIPATH_ACCESS_TOKEN") + return UiPath() + + +@pytest.fixture(scope="module") +def folder_key() -> str: + return _require_env("UIPATH_FOLDER_KEY") + + +@pytest.fixture(scope="module") +def memory_index(sdk: UiPath, folder_key: str): # noqa: ANN201 + """Create a test memory index and clean it up after all tests.""" + unique_name = f"sdk-e2e-test-{uuid.uuid4().hex[:8]}" + index = sdk.memory.create( + name=unique_name, + description="Created by E2E test — safe to delete", + folder_key=folder_key, + ) + yield index + + +class TestMemoryServiceE2E: + """E2E tests for MemoryService lifecycle. + + Requires: UIPATH_URL, UIPATH_ACCESS_TOKEN, UIPATH_FOLDER_KEY + """ + + # ── Index CRUD (ECS) ────────────────────────────────────────── + + def test_create_index(self, memory_index: MemorySpace) -> None: + """Verify index creation returns a well-formed MemorySpace.""" + assert memory_index.id, "Index ID should be set" + assert memory_index.name.startswith("sdk-e2e-test-") + assert memory_index.folder_key, "Folder key should be populated" + assert memory_index.memories_count == 0 + + def test_list_indexes( + self, + sdk: UiPath, + memory_index: MemorySpace, + folder_key: str, + ) -> None: + """Verify list with OData filter returns our index.""" + result = sdk.memory.list( + filter=f"Name eq '{memory_index.name}'", + folder_key=folder_key, + ) + assert isinstance(result, MemorySpaceListResponse) + names = [idx.name for idx in result.value] + assert memory_index.name in names + + # ── Search (LLMOps) ────────────────────────────────────────── + + def test_search_empty_index( + self, + sdk: UiPath, + memory_index: MemorySpace, + folder_key: str, + ) -> None: + """Search an empty index — should return empty results and systemPromptInjection.""" + request = MemorySearchRequest( + fields=[ + SearchField( + key_path=["input"], + value="test query", + ) + ], + settings=SearchSettings( + threshold=0.0, + result_count=5, + search_mode=SearchMode.Hybrid, + ), + definition_system_prompt="You are a helpful assistant.", + ) + result = sdk.memory.search( + memory_space_id=memory_index.id, + request=request, + folder_key=folder_key, + ) + assert isinstance(result, MemorySearchResponse) + assert isinstance(result.results, list) + assert isinstance(result.metadata, dict) + assert isinstance(result.system_prompt_injection, str) + + # ── Escalation search (LLMOps) ──────────────────────────────── + + def test_escalation_search_empty_index( + self, + sdk: UiPath, + memory_index: MemorySpace, + folder_key: str, + ) -> None: + """Search escalation memory on empty index — should return valid response.""" + request = MemorySearchRequest( + fields=[ + SearchField( + key_path=["input"], + value="test escalation query", + ) + ], + settings=SearchSettings( + threshold=0.0, + result_count=5, + search_mode=SearchMode.Hybrid, + ), + definition_system_prompt="You are a helpful assistant.", + ) + result = sdk.memory.escalation_search( + memory_space_id=memory_index.id, + request=request, + folder_key=folder_key, + ) + assert isinstance(result, EscalationMemorySearchResponse) diff --git a/packages/uipath-platform/tests/services/test_pii_detection_service.py b/packages/uipath-platform/tests/services/test_pii_detection_service.py new file mode 100644 index 000000000..2bb424607 --- /dev/null +++ b/packages/uipath-platform/tests/services/test_pii_detection_service.py @@ -0,0 +1,264 @@ +"""Tests for PiiDetectionService.""" + +import json +from typing import Any + +import httpx +import pytest +from pytest_httpx import HTTPXMock + +from uipath.platform import UiPathApiConfig, UiPathExecutionContext +from uipath.platform.pii_detection import ( + PiiDetectionRequest, + PiiDetectionResponse, + PiiDetectionService, + PiiDocument, + PiiEntityThreshold, + PiiFile, +) + + +@pytest.fixture +def service( + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, +) -> PiiDetectionService: + return PiiDetectionService(config=config, execution_context=execution_context) + + +@pytest.fixture +def sample_response_json() -> dict[str, Any]: + return { + "response": [ + { + "id": "user-prompt", + "role": "user", + "maskedDocument": "Contact [Person-1]", + "initialDocument": "Contact Alison", + "piiEntities": [ + { + "piiText": "Alison", + "replacementText": "[Person-1]", + "piiType": "Person", + "offset": 8, + "confidenceScore": 0.99, + } + ], + } + ], + "files": [ + { + "fileName": "doc.pdf", + "fileUrl": "https://blob.example.com/redacted/doc.pdf", + "piiEntities": [ + { + "piiText": "alice@example.com", + "replacementText": "[Email-1]", + "piiType": "Email", + "offset": 100, + "confidenceScore": 0.88, + } + ], + } + ], + } + + +class TestPiiDetectionService: + """Test PiiDetectionService functionality.""" + + class TestDetectPii: + """Test detect_pii (sync).""" + + def test_returns_typed_response( + self, + httpx_mock: HTTPXMock, + service: PiiDetectionService, + base_url: str, + org: str, + tenant: str, + sample_response_json: dict[str, Any], + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/llmopstenant_/api/pii-detection", + status_code=200, + json=sample_response_json, + ) + + request = PiiDetectionRequest( + documents=[ + PiiDocument( + id="user-prompt", role="user", document="Contact Alison" + ) + ] + ) + result = service.detect_pii(request) + + assert isinstance(result, PiiDetectionResponse) + assert len(result.response) == 1 + assert result.response[0].masked_document == "Contact [Person-1]" + assert len(result.files) == 1 + assert result.files[0].file_name == "doc.pdf" + assert result.files[0].pii_entities[0].replacement_text == "[Email-1]" + + class TestDetectPiiAsync: + """Test detect_pii_async.""" + + async def test_returns_typed_response( + self, + httpx_mock: HTTPXMock, + service: PiiDetectionService, + base_url: str, + org: str, + tenant: str, + sample_response_json: dict[str, Any], + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/llmopstenant_/api/pii-detection", + status_code=200, + json=sample_response_json, + ) + + request = PiiDetectionRequest( + files=[ + PiiFile( + file_name="doc.pdf", + file_url="https://input.example.com/doc.pdf", + file_type="pdf", + ) + ] + ) + result = await service.detect_pii_async(request) + + assert isinstance(result, PiiDetectionResponse) + assert ( + result.files[0].file_url == "https://blob.example.com/redacted/doc.pdf" + ) + + async def test_request_payload_uses_aliases( + self, + httpx_mock: HTTPXMock, + service: PiiDetectionService, + base_url: str, + org: str, + tenant: str, + sample_response_json: dict[str, Any], + ) -> None: + captured_request: httpx.Request | None = None + + def capture(request: httpx.Request) -> httpx.Response: + nonlocal captured_request + captured_request = request + return httpx.Response(status_code=200, json=sample_response_json) + + httpx_mock.add_callback( + method="POST", + url=f"{base_url}{org}{tenant}/llmopstenant_/api/pii-detection", + callback=capture, + ) + + request = PiiDetectionRequest( + documents=[ + PiiDocument(id="user-prompt", role="user", document="Hello") + ], + files=[ + PiiFile( + file_name="doc.pdf", + file_url="https://input.example.com/doc.pdf", + file_type="pdf", + ) + ], + language_code="en", + confidence_threshold=0.5, + entity_thresholds=[ + PiiEntityThreshold(category="Person", confidence_threshold=0.7), + ], + ) + await service.detect_pii_async(request) + + assert captured_request is not None + payload = json.loads(captured_request.content) + + # Top-level uses camelCase aliases + assert "documents" in payload + assert "files" in payload + assert "languageCode" in payload + assert "confidenceThreshold" in payload + assert "entityThresholds" in payload + + # File uses camelCase aliases + assert payload["files"][0]["fileName"] == "doc.pdf" + assert payload["files"][0]["fileUrl"] == "https://input.example.com/doc.pdf" + assert payload["files"][0]["fileType"] == "pdf" + + # Entity threshold uses kebab-case aliases + threshold = payload["entityThresholds"][0] + assert threshold["pii-entity-category"] == "Person" + assert threshold["pii-entity-confidence-threshold"] == 0.7 + + async def test_request_excludes_none_fields( + self, + httpx_mock: HTTPXMock, + service: PiiDetectionService, + base_url: str, + org: str, + tenant: str, + sample_response_json: dict[str, Any], + ) -> None: + captured_request: httpx.Request | None = None + + def capture(request: httpx.Request) -> httpx.Response: + nonlocal captured_request + captured_request = request + return httpx.Response(status_code=200, json=sample_response_json) + + httpx_mock.add_callback( + method="POST", + url=f"{base_url}{org}{tenant}/llmopstenant_/api/pii-detection", + callback=capture, + ) + + # Only documents set; other optional fields should be omitted + request = PiiDetectionRequest( + documents=[PiiDocument(id="user-prompt", role="user", document="Hello")] + ) + await service.detect_pii_async(request) + + assert captured_request is not None + payload = json.loads(captured_request.content) + assert "files" not in payload + assert "languageCode" not in payload + assert "confidenceThreshold" not in payload + assert "entityThresholds" not in payload + + async def test_url_is_tenant_scoped( + self, + httpx_mock: HTTPXMock, + service: PiiDetectionService, + base_url: str, + org: str, + tenant: str, + sample_response_json: dict[str, Any], + ) -> None: + captured_request: httpx.Request | None = None + + def capture(request: httpx.Request) -> httpx.Response: + nonlocal captured_request + captured_request = request + return httpx.Response(status_code=200, json=sample_response_json) + + httpx_mock.add_callback( + method="POST", + url=f"{base_url}{org}{tenant}/llmopstenant_/api/pii-detection", + callback=capture, + ) + + request = PiiDetectionRequest( + documents=[PiiDocument(id="user-prompt", role="user", document="Hello")] + ) + await service.detect_pii_async(request) + + assert captured_request is not None + assert org.strip("/") in captured_request.url.path + assert tenant.strip("/") in captured_request.url.path + assert "/llmopstenant_/api/pii-detection" in captured_request.url.path diff --git a/packages/uipath-platform/tests/services/test_pii_utilities.py b/packages/uipath-platform/tests/services/test_pii_utilities.py new file mode 100644 index 000000000..751897609 --- /dev/null +++ b/packages/uipath-platform/tests/services/test_pii_utilities.py @@ -0,0 +1,179 @@ +"""Tests for PII rehydration utilities.""" + +from uipath.platform.pii_detection import ( + PiiDetectionResponse, + PiiDocumentResult, + PiiEntity, + PiiFileResult, + rehydrate_from_pii_entities, + rehydrate_from_pii_response, +) + + +def _entity( + pii_text: str, + replacement_text: str, + pii_type: str = "Person", + offset: int = 0, + confidence_score: float = 0.9, +) -> PiiEntity: + return PiiEntity( + pii_text=pii_text, + replacement_text=replacement_text, + pii_type=pii_type, + offset=offset, + confidence_score=confidence_score, + ) + + +class TestRehydrateFromPiiEntities: + """Test rehydrate_from_pii_entities.""" + + def test_empty_text_returns_empty(self) -> None: + assert rehydrate_from_pii_entities("", [_entity("Alice", "[Person-1]")]) == "" + + def test_no_entities_returns_text_unchanged(self) -> None: + text = "Hello [Person-1]" + assert rehydrate_from_pii_entities(text, []) == text + + def test_replaces_single_placeholder(self) -> None: + result = rehydrate_from_pii_entities( + "Hello [Person-1]", [_entity("Alice", "[Person-1]")] + ) + assert result == "Hello Alice" + + def test_replaces_multiple_placeholders(self) -> None: + result = rehydrate_from_pii_entities( + "Contact [Person-1] at [Email-1]", + [ + _entity("Alice", "[Person-1]"), + _entity("alice@example.com", "[Email-1]", pii_type="Email"), + ], + ) + assert result == "Contact Alice at alice@example.com" + + def test_longer_placeholders_replaced_first(self) -> None: + """[Person-10] must be rehydrated before [Person-1] to avoid partial match.""" + result = rehydrate_from_pii_entities( + "[Person-1] and [Person-10]", + [ + _entity("Alice", "[Person-1]"), + _entity("Zara", "[Person-10]"), + ], + ) + assert result == "Alice and Zara" + + def test_case_insensitive_placeholder_match(self) -> None: + result = rehydrate_from_pii_entities( + "Hello [person-1]", [_entity("Alice", "[Person-1]")] + ) + assert result == "Hello Alice" + + def test_replaces_bracketless_variant(self) -> None: + """The LLM may drop brackets; bracketless variant should still be replaced.""" + result = rehydrate_from_pii_entities( + "Hello Person-1", [_entity("Alice", "[Person-1]")] + ) + assert result == "Hello Alice" + + def test_skips_entities_with_empty_replacement_text(self) -> None: + result = rehydrate_from_pii_entities( + "Hello [Person-1]", + [ + _entity("Ignored", ""), + _entity("Alice", "[Person-1]"), + ], + ) + assert result == "Hello Alice" + + def test_skips_entities_with_empty_pii_text(self) -> None: + result = rehydrate_from_pii_entities( + "Hello [Person-1]", + [_entity("", "[Person-1]")], + ) + assert result == "Hello [Person-1]" + + def test_preserves_non_placeholder_content(self) -> None: + result = rehydrate_from_pii_entities( + "The meeting with [Person-1] is at 3pm in the boardroom.", + [_entity("Alice", "[Person-1]")], + ) + assert result == "The meeting with Alice is at 3pm in the boardroom." + + def test_pii_text_with_special_characters(self) -> None: + """Special chars in PII text must not break regex substitution.""" + result = rehydrate_from_pii_entities( + "Visit [URL-1]", + [_entity("https://example.com/path?q=1&x=2", "[URL-1]", pii_type="URL")], + ) + assert result == "Visit https://example.com/path?q=1&x=2" + + def test_regex_special_chars_in_replacement_text(self) -> None: + """Regex special chars in the placeholder must be escaped for the pattern.""" + result = rehydrate_from_pii_entities( + "Hello [Person.1]", + [_entity("Alice", "[Person.1]")], + ) + assert result == "Hello Alice" + + def test_pii_text_inserted_verbatim_not_json_escaped(self) -> None: + """PII values are plain text: quotes, newlines and backslashes are + inserted as-is, never JSON-escaped.""" + pii = 'Bob "The Boss"\nC:\\Users\\bob' + result = rehydrate_from_pii_entities( + "Name: [Person-1]", [_entity(pii, "[Person-1]")] + ) + assert result == f"Name: {pii}" + + +class TestRehydrateFromPiiResponse: + """Test rehydrate_from_pii_response.""" + + def test_merges_document_and_file_entities(self) -> None: + response = PiiDetectionResponse( + response=[ + PiiDocumentResult( + id="user-prompt", + role="user", + masked_document="Hi [Person-1]", + initial_document="Hi Alice", + pii_entities=[_entity("Alice", "[Person-1]")], + ) + ], + files=[ + PiiFileResult( + file_name="doc.pdf", + file_url="https://example.com/doc.pdf", + pii_entities=[ + _entity("bob@example.com", "[Email-1]", pii_type="Email") + ], + ) + ], + ) + + result = rehydrate_from_pii_response( + "From [Person-1]: contact [Email-1]", response + ) + assert result == "From Alice: contact bob@example.com" + + def test_file_only_entity_is_rehydrated(self) -> None: + """Entities detected in files (not prompts) must also rehydrate.""" + response = PiiDetectionResponse( + response=[], + files=[ + PiiFileResult( + file_name="doc.pdf", + file_url="https://example.com/doc.pdf", + pii_entities=[ + _entity("alice@example.com", "[Email-1]", pii_type="Email") + ], + ) + ], + ) + + result = rehydrate_from_pii_response("Email is [Email-1]", response) + assert result == "Email is alice@example.com" + + def test_empty_response_returns_text_unchanged(self) -> None: + response = PiiDetectionResponse(response=[], files=[]) + assert rehydrate_from_pii_response("No PII here", response) == "No PII here" diff --git a/packages/uipath-platform/tests/services/test_processes_service.py b/packages/uipath-platform/tests/services/test_processes_service.py index 85b2e3691..054313a1c 100644 --- a/packages/uipath-platform/tests/services/test_processes_service.py +++ b/packages/uipath-platform/tests/services/test_processes_service.py @@ -5,7 +5,7 @@ from pytest_httpx import HTTPXMock from uipath.platform import UiPathApiConfig, UiPathExecutionContext -from uipath.platform.common.constants import HEADER_USER_AGENT +from uipath.platform.constants import HEADER_USER_AGENT from uipath.platform.orchestrator import Job from uipath.platform.orchestrator._attachments_service import AttachmentsService from uipath.platform.orchestrator._processes_service import ProcessesService @@ -79,6 +79,7 @@ def test_invoke( "startInfo": { "ReleaseName": process_name, "InputArguments": json.dumps(input_arguments), + "Source": "AgentService", } }, separators=(",", ":"), @@ -139,6 +140,7 @@ def test_invoke_without_input_arguments( "startInfo": { "ReleaseName": process_name, "InputArguments": "{}", + "Source": "AgentService", } }, separators=(",", ":"), @@ -300,6 +302,7 @@ async def test_invoke_async( "startInfo": { "ReleaseName": process_name, "InputArguments": json.dumps(input_arguments), + "Source": "AgentService", } }, separators=(",", ":"), @@ -361,6 +364,7 @@ async def test_invoke_async_without_input_arguments( "startInfo": { "ReleaseName": process_name, "InputArguments": "{}", + "Source": "AgentService", } }, separators=(",", ":"), @@ -471,3 +475,96 @@ async def test_invoke_async_over_10k_limit_input( job_request.headers[HEADER_USER_AGENT] == f"UiPath.Python.Sdk/UiPath.Python.Sdk.Activities.ProcessesService.invoke_async/{version}" ) + + def test_invoke_with_run_as_me_true( + self, + httpx_mock: HTTPXMock, + service: ProcessesService, + base_url: str, + org: str, + tenant: str, + ) -> None: + process_name = "test-process" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/odata/Jobs/UiPath.Server.Configuration.OData.StartJobs", + status_code=200, + json={ + "value": [ + { + "Key": "test-job-key", + "State": "Running", + "Id": 123, + "FolderKey": "test-folder-key", + } + ] + }, + ) + + service.invoke(process_name, run_as_me=True) + + sent_request = httpx_mock.get_request() + assert sent_request is not None + payload = json.loads(sent_request.content.decode("utf-8")) + assert payload["startInfo"]["RunAsMe"] is True + + def test_invoke_with_run_as_me_false( + self, + httpx_mock: HTTPXMock, + service: ProcessesService, + base_url: str, + org: str, + tenant: str, + ) -> None: + process_name = "test-process" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/odata/Jobs/UiPath.Server.Configuration.OData.StartJobs", + status_code=200, + json={ + "value": [ + { + "Key": "test-job-key", + "State": "Running", + "Id": 123, + "FolderKey": "test-folder-key", + } + ] + }, + ) + + service.invoke(process_name, run_as_me=False) + + sent_request = httpx_mock.get_request() + assert sent_request is not None + payload = json.loads(sent_request.content.decode("utf-8")) + assert payload["startInfo"]["RunAsMe"] is False + + def test_invoke_without_run_as_me_excludes_from_payload( + self, + httpx_mock: HTTPXMock, + service: ProcessesService, + base_url: str, + org: str, + tenant: str, + ) -> None: + process_name = "test-process" + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/odata/Jobs/UiPath.Server.Configuration.OData.StartJobs", + status_code=200, + json={ + "value": [ + { + "Key": "test-job-key", + "State": "Running", + "Id": 123, + "FolderKey": "test-folder-key", + } + ] + }, + ) + + service.invoke(process_name) + + sent_request = httpx_mock.get_request() + assert sent_request is not None + payload = json.loads(sent_request.content.decode("utf-8")) + assert "RunAsMe" not in payload["startInfo"] diff --git a/packages/uipath-platform/tests/services/test_queues_service.py b/packages/uipath-platform/tests/services/test_queues_service.py index 51cfeaa95..da2d7b876 100644 --- a/packages/uipath-platform/tests/services/test_queues_service.py +++ b/packages/uipath-platform/tests/services/test_queues_service.py @@ -1,10 +1,11 @@ import json +from datetime import datetime, timezone import pytest from pytest_httpx import HTTPXMock from uipath.platform import UiPathApiConfig, UiPathExecutionContext -from uipath.platform.common.constants import ( +from uipath.platform.constants import ( HEADER_FOLDER_KEY, HEADER_FOLDER_PATH, HEADER_USER_AGENT, @@ -518,6 +519,39 @@ async def test_create_item_with_reference_async( == f"UiPath.Python.Sdk/UiPath.Python.Sdk.Activities.QueuesService.create_item_async/{version}" ) + def test_create_item_with_datetime_fields( + self, + httpx_mock: HTTPXMock, + service: QueuesService, + base_url: str, + org: str, + tenant: str, + ) -> None: + defer = datetime(2026, 5, 1, 9, 0, 0, tzinfo=timezone.utc) + due = datetime(2026, 5, 2, 17, 30, 0, tzinfo=timezone.utc) + risk = datetime(2026, 5, 2, 12, 0, 0, tzinfo=timezone.utc) + queue_item = QueueItem( + priority=QueueItemPriority.NORMAL, + specific_content={"key": "value"}, + defer_date=defer, + due_date=due, + risk_sla_date=risk, + ) + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/odata/Queues/UiPathODataSvc.AddQueueItem", + status_code=200, + json={"Id": 1}, + ) + + service.create_item(queue_item, queue_name="test-queue") + + sent_request = httpx_mock.get_request() + assert sent_request is not None + body = json.loads(sent_request.content.decode()) + assert body["itemData"]["DeferDate"] == defer.isoformat() + assert body["itemData"]["DueDate"] == due.isoformat() + assert body["itemData"]["RiskSlaDate"] == risk.isoformat() + def test_create_transaction_item( self, httpx_mock: HTTPXMock, diff --git a/packages/uipath-platform/tests/services/test_reference_context.py b/packages/uipath-platform/tests/services/test_reference_context.py new file mode 100644 index 000000000..38be5aebf --- /dev/null +++ b/packages/uipath-platform/tests/services/test_reference_context.py @@ -0,0 +1,583 @@ +"""Tests for ReferenceContext, ReferenceContextAccessor, and span Context wiring.""" + +import json +from datetime import datetime +from unittest.mock import Mock + +import pytest +from opentelemetry.sdk.trace import Span as OTelSpan +from opentelemetry.trace import SpanContext, StatusCode + +from uipath.platform.common import _SpanUtils +from uipath.platform.common._reference_context import ( + ReferenceContext, + ReferenceContextAccessor, + ReferenceEntry, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_mock_span(attributes: dict[str, str] | None = None) -> Mock: + mock = Mock(spec=OTelSpan) + mock.get_span_context.return_value = SpanContext( + trace_id=0x123456789ABCDEF0123456789ABCDEF0, + span_id=0x0123456789ABCDEF, + is_remote=False, + ) + mock.name = "test-span" + mock.parent = None + mock.status.status_code = StatusCode.OK + mock.attributes = attributes or {} + mock.events = [] + mock.links = [] + now_ns = int(datetime.now().timestamp() * 1e9) + mock.start_time = now_ns + mock.end_time = now_ns + 1_000_000 + return mock + + +# --------------------------------------------------------------------------- +# ReferenceContext — immutability & copy-on-write +# --------------------------------------------------------------------------- + + +class TestReferenceContextImmutability: + def test_empty_singleton_is_falsy(self) -> None: + assert not ReferenceContext.Empty + + def test_add_returns_new_instance(self) -> None: + base = ReferenceContext.Empty + child = base.add("agent", "550e8400-e29b-41d4-a716-446655440001") + assert child is not base + + def test_original_unmodified_after_add(self) -> None: + base = ReferenceContext.Empty + base.add("agent", "550e8400-e29b-41d4-a716-446655440001") + assert len(base) == 0 + + def test_siblings_do_not_share_entries(self) -> None: + base = ReferenceContext.Empty.add( + "maestro", "550e8400-e29b-41d4-a716-446655440010", "2.0" + ) + child_a = base.add("agent", "550e8400-e29b-41d4-a716-446655440011") + child_b = base.add("agent", "550e8400-e29b-41d4-a716-446655440012") + + assert len(base) == 1 + assert len(child_a) == 2 + assert len(child_b) == 2 + assert child_a.entries[1].reference_id != child_b.entries[1].reference_id + + def test_equality_and_hash(self) -> None: + a = ReferenceContext.Empty.add("agent", "550e8400-e29b-41d4-a716-446655440001") + b = ReferenceContext.Empty.add("agent", "550e8400-e29b-41d4-a716-446655440001") + assert a == b + assert hash(a) == hash(b) + + def test_different_entries_not_equal(self) -> None: + a = ReferenceContext.Empty.add("agent", "550e8400-e29b-41d4-a716-446655440001") + b = ReferenceContext.Empty.add("agent", "550e8400-e29b-41d4-a716-446655440002") + assert a != b + + +# --------------------------------------------------------------------------- +# ReferenceContext.add — validation & UUID coercion +# --------------------------------------------------------------------------- + + +class TestReferenceContextAdd: + def test_add_with_string_uuid(self) -> None: + ctx = ReferenceContext.Empty.add( + "agent", "550e8400-e29b-41d4-a716-446655440001", "1.0" + ) + assert len(ctx) == 1 + e = ctx.entries[0] + assert e.service_type == "agent" + assert e.reference_id == "550e8400-e29b-41d4-a716-446655440001" + assert e.version == "1.0" + + def test_add_with_uuid_object(self) -> None: + import uuid + + uid = uuid.UUID("550e8400-e29b-41d4-a716-446655440001") + ctx = ReferenceContext.Empty.add("agent", uid) + assert ctx.entries[0].reference_id == str(uid) + + def test_add_without_version(self) -> None: + ctx = ReferenceContext.Empty.add( + "agent", "550e8400-e29b-41d4-a716-446655440001" + ) + assert ctx.entries[0].version is None + + def test_add_blank_version_normalised_to_none(self) -> None: + ctx = ReferenceContext.Empty.add( + "agent", "550e8400-e29b-41d4-a716-446655440001", " " + ) + assert ctx.entries[0].version is None + + def test_add_empty_service_type_raises(self) -> None: + with pytest.raises(ValueError, match="service_type"): + ReferenceContext.Empty.add("", "550e8400-e29b-41d4-a716-446655440001") + + def test_add_invalid_reference_id_type_raises(self) -> None: + with pytest.raises(TypeError, match="reference_id"): + ReferenceContext.Empty.add("agent", 12345) # type: ignore[arg-type] + + def test_entries_ordered_oldest_first(self) -> None: + ctx = ReferenceContext.Empty.add( + "maestro", "550e8400-e29b-41d4-a716-446655440010", "2.0" + ).add("agent", "550e8400-e29b-41d4-a716-446655440011") + assert ctx.entries[0].service_type == "maestro" + assert ctx.entries[1].service_type == "agent" + + +# --------------------------------------------------------------------------- +# ReferenceContext.to_wire_list +# --------------------------------------------------------------------------- + + +class TestToWireList: + def test_empty_produces_empty_list(self) -> None: + assert ReferenceContext.Empty.to_wire_list() == [] + + def test_single_entry_with_version(self) -> None: + ctx = ReferenceContext.Empty.add( + "agent", "550e8400-e29b-41d4-a716-446655440001", "1.0.0" + ) + wire = ctx.to_wire_list() + assert wire == [ + { + "serviceType": "agent", + "referenceId": "550e8400-e29b-41d4-a716-446655440001", + "version": "1.0.0", + } + ] + + def test_single_entry_without_version_omits_key(self) -> None: + ctx = ReferenceContext.Empty.add( + "agent", "550e8400-e29b-41d4-a716-446655440001" + ) + wire = ctx.to_wire_list() + assert "version" not in wire[0] + + def test_multiple_entries_order_preserved(self) -> None: + ctx = ReferenceContext.Empty.add( + "maestro", "550e8400-e29b-41d4-a716-446655440010", "2.1.0" + ).add("agent", "550e8400-e29b-41d4-a716-446655440011") + wire = ctx.to_wire_list() + assert len(wire) == 2 + assert wire[0]["serviceType"] == "maestro" + assert wire[0]["version"] == "2.1.0" + assert wire[1]["serviceType"] == "agent" + assert "version" not in wire[1] + + +# --------------------------------------------------------------------------- +# ReferenceContext.from_baggage_header / to_baggage_header_value round-trip +# --------------------------------------------------------------------------- + + +class TestBaggageHeaderRoundTrip: + def test_round_trip_single_entry_with_version(self) -> None: + ctx = ReferenceContext.Empty.add( + "agent", "550e8400-e29b-41d4-a716-446655440001", "1.0" + ) + assert ( + ReferenceContext.from_baggage_header(ctx.to_baggage_header_value()) == ctx + ) + + def test_round_trip_multiple_entries(self) -> None: + ctx = ReferenceContext.Empty.add( + "maestro", "550e8400-e29b-41d4-a716-446655440010", "2.0" + ).add("agent", "550e8400-e29b-41d4-a716-446655440011") + assert ( + ReferenceContext.from_baggage_header(ctx.to_baggage_header_value()) == ctx + ) + + def test_empty_header_returns_empty(self) -> None: + assert ReferenceContext.from_baggage_header("") == ReferenceContext.Empty + assert ReferenceContext.from_baggage_header(None) == ReferenceContext.Empty + + def test_malformed_entry_skipped_silently(self) -> None: + # Only the second entry is valid + header = "not-a-valid-entry,ref.type=agent;ref.id=550e8400-e29b-41d4-a716-446655440001" + ctx = ReferenceContext.from_baggage_header(header) + assert len(ctx) == 1 + assert ctx.entries[0].service_type == "agent" + + def test_entry_with_invalid_uuid_skipped(self) -> None: + header = "ref.type=agent;ref.id=not-a-uuid" + ctx = ReferenceContext.from_baggage_header(header) + assert ctx == ReferenceContext.Empty + + def test_entry_missing_type_skipped(self) -> None: + header = "ref.id=550e8400-e29b-41d4-a716-446655440001" + ctx = ReferenceContext.from_baggage_header(header) + assert ctx == ReferenceContext.Empty + + def test_empty_context_produces_empty_header(self) -> None: + assert ReferenceContext.Empty.to_baggage_header_value() == "" + + +# --------------------------------------------------------------------------- +# ReferenceContextAccessor — ContextVar semantics +# --------------------------------------------------------------------------- + + +class TestReferenceContextAccessor: + def setup_method(self) -> None: + # Ensure clean state before each test + current = ReferenceContextAccessor.get() + if current is not None: + token = ReferenceContextAccessor.set(None) + # immediately reset to avoid polluting other tests + ReferenceContextAccessor.reset(token) + + def test_default_is_none(self) -> None: + assert ReferenceContextAccessor.get() is None + + def test_set_and_get(self) -> None: + ctx = ReferenceContext.Empty.add( + "agent", "550e8400-e29b-41d4-a716-446655440001" + ) + token = ReferenceContextAccessor.set(ctx) + try: + assert ReferenceContextAccessor.get() == ctx + finally: + ReferenceContextAccessor.reset(token) + + def test_reset_restores_prior_value(self) -> None: + ctx_a = ReferenceContext.Empty.add( + "agent", "550e8400-e29b-41d4-a716-446655440001" + ) + ctx_b = ctx_a.add("langgraph", "550e8400-e29b-41d4-a716-446655440002") + + token_a = ReferenceContextAccessor.set(ctx_a) + token_b = ReferenceContextAccessor.set(ctx_b) + + assert ReferenceContextAccessor.get() == ctx_b + ReferenceContextAccessor.reset(token_b) + assert ReferenceContextAccessor.get() == ctx_a + ReferenceContextAccessor.reset(token_a) + assert ReferenceContextAccessor.get() is None + + +# --------------------------------------------------------------------------- +# UiPathSpan.context wiring via otel_span_to_uipath_span +# --------------------------------------------------------------------------- + + +class TestContextWiring: + def test_context_absent_when_no_reference_context_set( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("UIPATH_ORGANIZATION_ID", "test-org") + span = _SpanUtils.otel_span_to_uipath_span(_make_mock_span()) + assert span.context is None + assert "Context" not in span.to_dict() + + def test_context_present_when_reference_context_set( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("UIPATH_ORGANIZATION_ID", "test-org") + ref_ctx = ReferenceContext.Empty.add( + "agent", "550e8400-e29b-41d4-a716-446655440001", "1.0" + ) + mock = _make_mock_span( + attributes={ + "uipath.reference_hierarchy": json.dumps(ref_ctx.to_wire_list()) + } + ) + span = _SpanUtils.otel_span_to_uipath_span(mock) + assert span.context == { + "referenceHierarchy": [ + { + "serviceType": "agent", + "referenceId": "550e8400-e29b-41d4-a716-446655440001", + "version": "1.0", + } + ] + } + wire = span.to_dict() + assert "Context" in wire + assert wire["Context"]["referenceHierarchy"][0]["serviceType"] == "agent" + + def test_context_carries_full_hierarchy( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("UIPATH_ORGANIZATION_ID", "test-org") + ref_ctx = ReferenceContext.Empty.add( + "maestro", "550e8400-e29b-41d4-a716-446655440010", "2.0" + ).add("agent", "550e8400-e29b-41d4-a716-446655440011") + mock = _make_mock_span( + attributes={ + "uipath.reference_hierarchy": json.dumps(ref_ctx.to_wire_list()) + } + ) + wire = _SpanUtils.otel_span_to_uipath_span(mock).to_dict() + hierarchy = wire["Context"]["referenceHierarchy"] + assert len(hierarchy) == 2 + assert hierarchy[0]["serviceType"] == "maestro" + assert hierarchy[0]["version"] == "2.0" + assert hierarchy[1]["serviceType"] == "agent" + assert "version" not in hierarchy[1] + + def test_reference_hierarchy_not_in_attributes_field( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("UIPATH_ORGANIZATION_ID", "test-org") + ref_ctx = ReferenceContext.Empty.add( + "agent", "550e8400-e29b-41d4-a716-446655440001" + ) + mock = _make_mock_span( + attributes={ + "uipath.reference_hierarchy": json.dumps(ref_ctx.to_wire_list()) + } + ) + wire = _SpanUtils.otel_span_to_uipath_span(mock).to_dict() + attributes = json.loads(wire["Attributes"]) + assert "uipath.reference_hierarchy" not in attributes + + +# --------------------------------------------------------------------------- +# ReferenceEntry — frozen dataclass +# --------------------------------------------------------------------------- + + +class TestReferenceEntry: + def test_frozen_raises_on_mutation(self) -> None: + entry = ReferenceEntry( + service_type="agent", reference_id="550e8400-e29b-41d4-a716-446655440001" + ) + with pytest.raises((AttributeError, TypeError)): + entry.service_type = "other" # type: ignore[misc] + + def test_equality_by_value(self) -> None: + a = ReferenceEntry( + service_type="agent", + reference_id="550e8400-e29b-41d4-a716-446655440001", + version="1.0", + ) + b = ReferenceEntry( + service_type="agent", + reference_id="550e8400-e29b-41d4-a716-446655440001", + version="1.0", + ) + assert a == b + + def test_version_defaults_to_none(self) -> None: + entry = ReferenceEntry( + service_type="agent", reference_id="550e8400-e29b-41d4-a716-446655440001" + ) + assert entry.version is None + + +# --------------------------------------------------------------------------- +# ReferenceContext — bool, iter, len +# --------------------------------------------------------------------------- + + +class TestReferenceContextProtocol: + def test_non_empty_context_is_truthy(self) -> None: + ctx = ReferenceContext.Empty.add( + "agent", "550e8400-e29b-41d4-a716-446655440001" + ) + assert bool(ctx) + + def test_iter_yields_entries_in_order(self) -> None: + ctx = ReferenceContext.Empty.add( + "maestro", "550e8400-e29b-41d4-a716-446655440010" + ).add("agent", "550e8400-e29b-41d4-a716-446655440011") + types = [e.service_type for e in ctx] + assert types == ["maestro", "agent"] + + def test_len_matches_entry_count(self) -> None: + ctx = ( + ReferenceContext.Empty.add( + "maestro", "550e8400-e29b-41d4-a716-446655440010" + ) + .add("agent", "550e8400-e29b-41d4-a716-446655440011") + .add("langgraph", "550e8400-e29b-41d4-a716-446655440012") + ) + assert len(ctx) == 3 + + +# --------------------------------------------------------------------------- +# ReferenceContext.add — additional validation cases +# --------------------------------------------------------------------------- + + +class TestReferenceContextAddValidation: + def test_whitespace_only_service_type_raises(self) -> None: + with pytest.raises(ValueError, match="service_type"): + ReferenceContext.Empty.add(" ", "550e8400-e29b-41d4-a716-446655440001") + + def test_invalid_uuid_string_raises_value_error(self) -> None: + with pytest.raises(ValueError, match="reference_id"): + ReferenceContext.Empty.add("agent", "not-a-valid-uuid") + + +# --------------------------------------------------------------------------- +# ReferenceContext.from_baggage_header — additional parsing cases +# --------------------------------------------------------------------------- + + +class TestFromBaggageHeaderParsing: + def test_whitespace_only_header_returns_empty(self) -> None: + assert ReferenceContext.from_baggage_header(" ") == ReferenceContext.Empty + + def test_entry_missing_ref_id_skipped(self) -> None: + header = "ref.type=agent" + ctx = ReferenceContext.from_baggage_header(header) + assert ctx == ReferenceContext.Empty + + def test_version_key_parsed_correctly(self) -> None: + header = ( + "ref.type=maestro;ref.id=550e8400-e29b-41d4-a716-446655440010;ref.v=2.1.0" + ) + ctx = ReferenceContext.from_baggage_header(header) + assert len(ctx) == 1 + assert ctx.entries[0].version == "2.1.0" + + def test_whitespace_only_entries_between_commas_skipped(self) -> None: + # ",, ," should not produce entries + header = "ref.type=agent;ref.id=550e8400-e29b-41d4-a716-446655440001, , " + ctx = ReferenceContext.from_baggage_header(header) + assert len(ctx) == 1 + + def test_mixed_valid_and_invalid_keeps_valid(self) -> None: + header = ( + "ref.type=agent;ref.id=not-a-uuid," + "ref.type=maestro;ref.id=550e8400-e29b-41d4-a716-446655440010" + ) + ctx = ReferenceContext.from_baggage_header(header) + assert len(ctx) == 1 + assert ctx.entries[0].service_type == "maestro" + + +# --------------------------------------------------------------------------- +# ReferenceContext.to_baggage_header_value — format checks +# --------------------------------------------------------------------------- + + +class TestToBaggageHeaderValue: + def test_single_entry_with_version_format(self) -> None: + ctx = ReferenceContext.Empty.add( + "agent", "550e8400-e29b-41d4-a716-446655440001", "1.0" + ) + value = ctx.to_baggage_header_value() + assert ( + value + == "ref.type=agent;ref.id=550e8400-e29b-41d4-a716-446655440001;ref.v=1.0" + ) + + def test_single_entry_without_version_format(self) -> None: + ctx = ReferenceContext.Empty.add( + "agent", "550e8400-e29b-41d4-a716-446655440001" + ) + value = ctx.to_baggage_header_value() + assert value == "ref.type=agent;ref.id=550e8400-e29b-41d4-a716-446655440001" + assert "ref.v" not in value + + def test_multiple_entries_comma_separated(self) -> None: + ctx = ReferenceContext.Empty.add( + "maestro", "550e8400-e29b-41d4-a716-446655440010", "2.0" + ).add("agent", "550e8400-e29b-41d4-a716-446655440011") + value = ctx.to_baggage_header_value() + parts = value.split(",") + assert len(parts) == 2 + assert parts[0].startswith("ref.type=maestro") + assert parts[1].startswith("ref.type=agent") + + +# --------------------------------------------------------------------------- +# ReferenceContextAccessor — async propagation +# --------------------------------------------------------------------------- + + +class TestReferenceContextAccessorAsync: + @pytest.mark.asyncio + async def test_context_propagates_across_await(self) -> None: + import asyncio + + ctx = ReferenceContext.Empty.add( + "agent", "550e8400-e29b-41d4-a716-446655440001" + ) + token = ReferenceContextAccessor.set(ctx) + try: + await asyncio.sleep(0) + assert ReferenceContextAccessor.get() == ctx + finally: + ReferenceContextAccessor.reset(token) + + @pytest.mark.asyncio + async def test_context_isolated_between_concurrent_tasks(self) -> None: + import asyncio + + ctx_a = ReferenceContext.Empty.add( + "agent", "550e8400-e29b-41d4-a716-446655440001" + ) + ctx_b = ReferenceContext.Empty.add( + "maestro", "550e8400-e29b-41d4-a716-446655440002" + ) + seen: dict[str, object] = {} + + async def task_a() -> None: + token = ReferenceContextAccessor.set(ctx_a) + await asyncio.sleep(0) + seen["a"] = ReferenceContextAccessor.get() + ReferenceContextAccessor.reset(token) + + async def task_b() -> None: + token = ReferenceContextAccessor.set(ctx_b) + await asyncio.sleep(0) + seen["b"] = ReferenceContextAccessor.get() + ReferenceContextAccessor.reset(token) + + await asyncio.gather(task_a(), task_b()) + assert seen["a"] == ctx_a + assert seen["b"] == ctx_b + + +# --------------------------------------------------------------------------- +# _inject_reference_hierarchy hook +# --------------------------------------------------------------------------- + + +class TestReferenceHierarchyHook: + def setup_method(self) -> None: + token = ReferenceContextAccessor.set(None) + ReferenceContextAccessor.reset(token) + + def test_hook_stamps_attribute_when_context_set(self) -> None: + from uipath.platform.common._span_utils import _inject_reference_hierarchy + + ref_ctx = ReferenceContext.Empty.add( + "agent", "550e8400-e29b-41d4-a716-446655440001" + ) + token = ReferenceContextAccessor.set(ref_ctx) + try: + mock_span = Mock() + _inject_reference_hierarchy(mock_span) + mock_span.set_attribute.assert_called_once() + key, value = mock_span.set_attribute.call_args[0] + assert key == "uipath.reference_hierarchy" + hierarchy = json.loads(value) + assert len(hierarchy) == 1 + assert hierarchy[0]["serviceType"] == "agent" + assert hierarchy[0]["referenceId"] == "550e8400-e29b-41d4-a716-446655440001" + finally: + ReferenceContextAccessor.reset(token) + + def test_hook_noop_when_context_not_set(self) -> None: + from uipath.platform.common._span_utils import _inject_reference_hierarchy + + token = ReferenceContextAccessor.set(None) + try: + mock_span = Mock() + _inject_reference_hierarchy(mock_span) + mock_span.set_attribute.assert_not_called() + finally: + ReferenceContextAccessor.reset(token) diff --git a/packages/uipath-platform/tests/services/test_remote_a2a_service.py b/packages/uipath-platform/tests/services/test_remote_a2a_service.py new file mode 100644 index 000000000..d6c56d6a2 --- /dev/null +++ b/packages/uipath-platform/tests/services/test_remote_a2a_service.py @@ -0,0 +1,239 @@ +from unittest.mock import AsyncMock, Mock, patch + +import pytest + +from uipath.platform import UiPathApiConfig, UiPathExecutionContext +from uipath.platform.agenthub._remote_a2a_service import RemoteA2aService +from uipath.platform.common._bindings import ( + GenericResourceOverwrite, + _resource_overwrites, +) +from uipath.platform.constants import HEADER_FOLDER_KEY +from uipath.platform.orchestrator._folder_service import FolderService + + +@pytest.fixture +def folders_service( + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + monkeypatch: pytest.MonkeyPatch, +) -> FolderService: + monkeypatch.setenv("UIPATH_FOLDER_KEY", "context-folder-key") + return FolderService(config=config, execution_context=execution_context) + + +@pytest.fixture +def service( + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + folders_service: FolderService, + monkeypatch: pytest.MonkeyPatch, +) -> RemoteA2aService: + monkeypatch.setenv("UIPATH_FOLDER_KEY", "context-folder-key") + return RemoteA2aService( + config=config, + execution_context=execution_context, + folders_service=folders_service, + ) + + +class TestRetrieveSpecFolderResolution: + def test_falls_back_to_folder_context_when_folder_path_missing( + self, service: RemoteA2aService + ) -> None: + """No folder_path (e.g. local debug) must not raise; it falls back to context.""" + spec = service._retrieve_spec(name="weather", folder_path=None) + + assert "remote-a2a-agents/weather" in str(spec.endpoint) + assert spec.headers[HEADER_FOLDER_KEY] == "context-folder-key" + + def test_resolves_explicit_folder_path( + self, service: RemoteA2aService, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr( + service._folders_service, + "retrieve_folder_key", + lambda folder_path: "resolved-folder-key", + ) + + spec = service._retrieve_spec(name="weather", folder_path="MyFolder") + + assert spec.headers[HEADER_FOLDER_KEY] == "resolved-folder-key" + + def test_encodes_display_name_in_lookup_path( + self, service: RemoteA2aService + ) -> None: + spec = service._retrieve_spec(name="Friendly Agent/Europe", folder_path=None) + + assert "remote-a2a-agents/Friendly%20Agent%2FEurope" in str(spec.endpoint) + + +class TestRetrieveByName: + def test_retrieves_by_display_name(self, service: RemoteA2aService) -> None: + response = Mock() + response.json.return_value = { + "name": "Friendly Agent/Europe", + "slug": "friendly-agent-europe", + } + + with patch.object(service, "request", return_value=response) as request: + agent = service.retrieve(name="Friendly Agent/Europe") + + assert agent.name == "Friendly Agent/Europe" + assert "remote-a2a-agents/Friendly%20Agent%2FEurope" in str( + request.call_args.kwargs["url"] + ) + + def test_applies_display_name_binding(self, service: RemoteA2aService) -> None: + response = Mock() + response.json.return_value = { + "name": "Replacement Agent", + "slug": "replacement-agent", + } + overwrite = GenericResourceOverwrite( + resource_type="remoteA2aAgent", + name="Replacement Agent", + folder_path="Replacement Folder", + ) + token = _resource_overwrites.set({"remoteA2aAgent.Original Agent": overwrite}) + + try: + with ( + patch.object(service, "request", return_value=response) as request, + patch.object( + service._folders_service, + "retrieve_folder_key", + return_value="replacement-folder-key", + ), + ): + service.retrieve(name="Original Agent") + finally: + _resource_overwrites.reset(token) + + assert "remote-a2a-agents/Replacement%20Agent" in str( + request.call_args.kwargs["url"] + ) + assert ( + request.call_args.kwargs["headers"][HEADER_FOLDER_KEY] + == "replacement-folder-key" + ) + + @pytest.mark.anyio + async def test_retrieves_by_display_name_async( + self, service: RemoteA2aService + ) -> None: + response = Mock() + response.json.return_value = { + "name": "Friendly Agent/Europe", + "slug": "friendly-agent-europe", + } + + with patch.object( + service, + "request_async", + new=AsyncMock(return_value=response), + ) as request: + agent = await service.retrieve_async(name="Friendly Agent/Europe") + + assert agent.name == "Friendly Agent/Europe" + assert "remote-a2a-agents/Friendly%20Agent%2FEurope" in str( + request.call_args.kwargs["url"] + ) + + +class TestRetrieveCompatibility: + def test_retrieves_by_positional_slug(self, service: RemoteA2aService) -> None: + response = Mock() + response.json.return_value = { + "name": "Friendly Agent", + "slug": "friendly-agent", + } + + with patch.object(service, "request", return_value=response) as request: + agent = service.retrieve("friendly-agent") + + assert agent.slug == "friendly-agent" + assert "remote-a2a-agents/friendly-agent" in str( + request.call_args.kwargs["url"] + ) + + def test_retrieves_by_slug_keyword(self, service: RemoteA2aService) -> None: + response = Mock() + response.json.return_value = { + "name": "Friendly Agent", + "slug": "friendly-agent", + } + + with patch.object(service, "request", return_value=response) as request: + agent = service.retrieve(slug="friendly-agent") + + assert agent.slug == "friendly-agent" + assert "remote-a2a-agents/friendly-agent" in str( + request.call_args.kwargs["url"] + ) + + def test_applies_legacy_slug_binding(self, service: RemoteA2aService) -> None: + response = Mock() + response.json.return_value = { + "name": "Replacement Agent", + "slug": "replacement-agent", + } + overwrite = GenericResourceOverwrite( + resource_type="remoteA2aAgent", + name="Replacement Agent", + folder_path="Replacement Folder", + ) + token = _resource_overwrites.set({"remoteA2aAgent.original-agent": overwrite}) + + try: + with ( + patch.object(service, "request", return_value=response) as request, + patch.object( + service._folders_service, + "retrieve_folder_key", + return_value="replacement-folder-key", + ), + ): + service.retrieve("original-agent") + finally: + _resource_overwrites.reset(token) + + assert "remote-a2a-agents/Replacement%20Agent" in str( + request.call_args.kwargs["url"] + ) + assert ( + request.call_args.kwargs["headers"][HEADER_FOLDER_KEY] + == "replacement-folder-key" + ) + + @pytest.mark.anyio + async def test_retrieves_by_positional_slug_async( + self, service: RemoteA2aService + ) -> None: + response = Mock() + response.json.return_value = { + "name": "Friendly Agent", + "slug": "friendly-agent", + } + + with patch.object( + service, + "request_async", + new=AsyncMock(return_value=response), + ) as request: + agent = await service.retrieve_async("friendly-agent") + + assert agent.slug == "friendly-agent" + assert "remote-a2a-agents/friendly-agent" in str( + request.call_args.kwargs["url"] + ) + + def test_rejects_name_and_slug(self, service: RemoteA2aService) -> None: + with pytest.raises( + ValueError, match="Specify either 'name' or 'slug', not both" + ): + service.retrieve("friendly-agent", name="Friendly Agent") + + def test_requires_name_or_slug(self, service: RemoteA2aService) -> None: + with pytest.raises(TypeError, match="Either 'name' or 'slug' must be provided"): + service.retrieve() diff --git a/packages/uipath-platform/tests/services/test_resource_catalog_service.py b/packages/uipath-platform/tests/services/test_resource_catalog_service.py index 3db6ee60e..fb2d9f4ea 100644 --- a/packages/uipath-platform/tests/services/test_resource_catalog_service.py +++ b/packages/uipath-platform/tests/services/test_resource_catalog_service.py @@ -5,7 +5,7 @@ from pytest_httpx import HTTPXMock from uipath.platform import UiPathApiConfig, UiPathExecutionContext -from uipath.platform.common.constants import HEADER_USER_AGENT +from uipath.platform.constants import HEADER_USER_AGENT from uipath.platform.orchestrator._folder_service import FolderService from uipath.platform.resource_catalog import ResourceType from uipath.platform.resource_catalog._resource_catalog_service import ( diff --git a/packages/uipath-platform/tests/services/test_semantic_proxy_service.py b/packages/uipath-platform/tests/services/test_semantic_proxy_service.py new file mode 100644 index 000000000..51b4f4895 --- /dev/null +++ b/packages/uipath-platform/tests/services/test_semantic_proxy_service.py @@ -0,0 +1,264 @@ +"""Tests for SemanticProxyService.""" + +import json +from typing import Any + +import httpx +import pytest +from pytest_httpx import HTTPXMock + +from uipath.platform import UiPathApiConfig, UiPathExecutionContext +from uipath.platform.semantic_proxy import ( + PiiDetectionRequest, + PiiDetectionResponse, + PiiDocument, + PiiEntityThreshold, + PiiFile, + SemanticProxyService, +) + + +@pytest.fixture +def service( + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, +) -> SemanticProxyService: + return SemanticProxyService(config=config, execution_context=execution_context) + + +@pytest.fixture +def sample_response_json() -> dict[str, Any]: + return { + "response": [ + { + "id": "user-prompt", + "role": "user", + "maskedDocument": "Contact [Person-1]", + "initialDocument": "Contact Alison", + "piiEntities": [ + { + "piiText": "Alison", + "replacementText": "[Person-1]", + "piiType": "Person", + "offset": 8, + "confidenceScore": 0.99, + } + ], + } + ], + "files": [ + { + "fileName": "doc.pdf", + "fileUrl": "https://blob.example.com/redacted/doc.pdf", + "piiEntities": [ + { + "piiText": "alice@example.com", + "replacementText": "[Email-1]", + "piiType": "Email", + "offset": 100, + "confidenceScore": 0.88, + } + ], + } + ], + } + + +class TestSemanticProxyService: + """Test SemanticProxyService functionality.""" + + class TestDetectPii: + """Test detect_pii (sync).""" + + def test_returns_typed_response( + self, + httpx_mock: HTTPXMock, + service: SemanticProxyService, + base_url: str, + org: str, + tenant: str, + sample_response_json: dict[str, Any], + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/semanticproxy_/api/pii-detection", + status_code=200, + json=sample_response_json, + ) + + request = PiiDetectionRequest( + documents=[ + PiiDocument( + id="user-prompt", role="user", document="Contact Alison" + ) + ] + ) + result = service.detect_pii(request) + + assert isinstance(result, PiiDetectionResponse) + assert len(result.response) == 1 + assert result.response[0].masked_document == "Contact [Person-1]" + assert len(result.files) == 1 + assert result.files[0].file_name == "doc.pdf" + assert result.files[0].pii_entities[0].replacement_text == "[Email-1]" + + class TestDetectPiiAsync: + """Test detect_pii_async.""" + + async def test_returns_typed_response( + self, + httpx_mock: HTTPXMock, + service: SemanticProxyService, + base_url: str, + org: str, + tenant: str, + sample_response_json: dict[str, Any], + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/semanticproxy_/api/pii-detection", + status_code=200, + json=sample_response_json, + ) + + request = PiiDetectionRequest( + files=[ + PiiFile( + file_name="doc.pdf", + file_url="https://input.example.com/doc.pdf", + file_type="pdf", + ) + ] + ) + result = await service.detect_pii_async(request) + + assert isinstance(result, PiiDetectionResponse) + assert ( + result.files[0].file_url == "https://blob.example.com/redacted/doc.pdf" + ) + + async def test_request_payload_uses_aliases( + self, + httpx_mock: HTTPXMock, + service: SemanticProxyService, + base_url: str, + org: str, + tenant: str, + sample_response_json: dict[str, Any], + ) -> None: + captured_request: httpx.Request | None = None + + def capture(request: httpx.Request) -> httpx.Response: + nonlocal captured_request + captured_request = request + return httpx.Response(status_code=200, json=sample_response_json) + + httpx_mock.add_callback( + method="POST", + url=f"{base_url}{org}{tenant}/semanticproxy_/api/pii-detection", + callback=capture, + ) + + request = PiiDetectionRequest( + documents=[ + PiiDocument(id="user-prompt", role="user", document="Hello") + ], + files=[ + PiiFile( + file_name="doc.pdf", + file_url="https://input.example.com/doc.pdf", + file_type="pdf", + ) + ], + language_code="en", + confidence_threshold=0.5, + entity_thresholds=[ + PiiEntityThreshold(category="Person", confidence_threshold=0.7), + ], + ) + await service.detect_pii_async(request) + + assert captured_request is not None + payload = json.loads(captured_request.content) + + # Top-level uses camelCase aliases + assert "documents" in payload + assert "files" in payload + assert "languageCode" in payload + assert "confidenceThreshold" in payload + assert "entityThresholds" in payload + + # File uses camelCase aliases + assert payload["files"][0]["fileName"] == "doc.pdf" + assert payload["files"][0]["fileUrl"] == "https://input.example.com/doc.pdf" + assert payload["files"][0]["fileType"] == "pdf" + + # Entity threshold uses kebab-case aliases + threshold = payload["entityThresholds"][0] + assert threshold["pii-entity-category"] == "Person" + assert threshold["pii-entity-confidence-threshold"] == 0.7 + + async def test_request_excludes_none_fields( + self, + httpx_mock: HTTPXMock, + service: SemanticProxyService, + base_url: str, + org: str, + tenant: str, + sample_response_json: dict[str, Any], + ) -> None: + captured_request: httpx.Request | None = None + + def capture(request: httpx.Request) -> httpx.Response: + nonlocal captured_request + captured_request = request + return httpx.Response(status_code=200, json=sample_response_json) + + httpx_mock.add_callback( + method="POST", + url=f"{base_url}{org}{tenant}/semanticproxy_/api/pii-detection", + callback=capture, + ) + + # Only documents set; other optional fields should be omitted + request = PiiDetectionRequest( + documents=[PiiDocument(id="user-prompt", role="user", document="Hello")] + ) + await service.detect_pii_async(request) + + assert captured_request is not None + payload = json.loads(captured_request.content) + assert "files" not in payload + assert "languageCode" not in payload + assert "confidenceThreshold" not in payload + assert "entityThresholds" not in payload + + async def test_url_is_tenant_scoped( + self, + httpx_mock: HTTPXMock, + service: SemanticProxyService, + base_url: str, + org: str, + tenant: str, + sample_response_json: dict[str, Any], + ) -> None: + captured_request: httpx.Request | None = None + + def capture(request: httpx.Request) -> httpx.Response: + nonlocal captured_request + captured_request = request + return httpx.Response(status_code=200, json=sample_response_json) + + httpx_mock.add_callback( + method="POST", + url=f"{base_url}{org}{tenant}/semanticproxy_/api/pii-detection", + callback=capture, + ) + + request = PiiDetectionRequest( + documents=[PiiDocument(id="user-prompt", role="user", document="Hello")] + ) + await service.detect_pii_async(request) + + assert captured_request is not None + assert org.strip("/") in captured_request.url.path + assert tenant.strip("/") in captured_request.url.path + assert "/semanticproxy_/api/pii-detection" in captured_request.url.path diff --git a/packages/uipath-platform/tests/services/test_service_url_overrides.py b/packages/uipath-platform/tests/services/test_service_url_overrides.py index cc038a3b9..271a7d183 100644 --- a/packages/uipath-platform/tests/services/test_service_url_overrides.py +++ b/packages/uipath-platform/tests/services/test_service_url_overrides.py @@ -4,6 +4,10 @@ inject_routing_headers, resolve_service_url, ) +from uipath.platform.constants import ( + HEADER_INTERNAL_ACCOUNT_ID, + HEADER_INTERNAL_TENANT_ID, +) class TestResolveServiceUrl: @@ -68,16 +72,16 @@ def test_injects_tenant_and_org(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("UIPATH_ORGANIZATION_ID", "org-456") headers: dict[str, str] = {} inject_routing_headers(headers) - assert headers["X-UiPath-Internal-TenantId"] == "tenant-123" - assert headers["X-UiPath-Internal-AccountId"] == "org-456" + assert headers[HEADER_INTERNAL_TENANT_ID] == "tenant-123" + assert headers[HEADER_INTERNAL_ACCOUNT_ID] == "org-456" def test_skips_missing_env_vars(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("UIPATH_TENANT_ID", raising=False) monkeypatch.delenv("UIPATH_ORGANIZATION_ID", raising=False) headers: dict[str, str] = {} inject_routing_headers(headers) - assert "X-UiPath-Internal-TenantId" not in headers - assert "X-UiPath-Internal-AccountId" not in headers + assert HEADER_INTERNAL_TENANT_ID not in headers + assert HEADER_INTERNAL_ACCOUNT_ID not in headers def test_does_not_overwrite_existing_headers( self, monkeypatch: pytest.MonkeyPatch @@ -87,4 +91,4 @@ def test_does_not_overwrite_existing_headers( headers: dict[str, str] = {"X-Custom": "keep-me"} inject_routing_headers(headers) assert headers["X-Custom"] == "keep-me" - assert headers["X-UiPath-Internal-TenantId"] == "tenant-123" + assert headers[HEADER_INTERNAL_TENANT_ID] == "tenant-123" diff --git a/packages/uipath-platform/tests/services/test_span_utils.py b/packages/uipath-platform/tests/services/test_span_utils.py index 80cd0d2db..b80d24e9d 100644 --- a/packages/uipath-platform/tests/services/test_span_utils.py +++ b/packages/uipath-platform/tests/services/test_span_utils.py @@ -1,13 +1,513 @@ import json +import logging import os from datetime import datetime from unittest.mock import Mock, patch import pytest +from opentelemetry import context as context_api from opentelemetry.sdk.trace import Span as OTelSpan +from opentelemetry.sdk.trace import SpanProcessor from opentelemetry.trace import SpanContext, StatusCode -from uipath.platform.common import UiPathSpan, _SpanUtils +from uipath.platform.common import ( + ReferenceHierarchySpanProcessor, + UiPathSpan, + _SpanUtils, +) +from uipath.platform.common._reference_context import ( + ReferenceContext, + ReferenceContextAccessor, +) +from uipath.platform.common._span_utils import ( + _SOURCE_BY_INT, + ExecutionType, + SpanSource, + SpanStatus, + VerbosityLevel, +) +from uipath.platform.constants import ( + ENV_PROJECT_KEY, + ENV_UIPATH_AGENT_ID, + ENV_UIPATH_PROJECT_ID, +) + + +@pytest.fixture(autouse=True) +def _clear_id_cache(): + """Isolate the process-global id cache between tests.""" + from uipath.platform.common._span_utils import _read_config_id + + _read_config_id.cache_clear() + yield + _read_config_id.cache_clear() + + +class TestStrEnums: + def test_span_status_string_values(self): + assert SpanStatus.UNSET == "Unset" + assert SpanStatus.OK == "Ok" + assert SpanStatus.ERROR == "Error" + assert SpanStatus.RUNNING == "Running" + assert SpanStatus.RESTRICTED == "Restricted" + assert SpanStatus.CANCELLED == "Cancelled" + + def test_span_source_string_values(self): + assert SpanSource.CODED_AGENTS == "CodedAgents" + assert SpanSource.AGENTS == "Agents" + assert SpanSource.PROCESS_ORCHESTRATION == "ProcessOrchestration" + assert SpanSource.API_WORKFLOWS == "ApiWorkflows" + assert SpanSource.ROBOTS == "Robots" + + def test_verbosity_level_string_values(self): + assert VerbosityLevel.VERBOSE == "Verbose" + assert VerbosityLevel.TRACE == "Trace" + assert VerbosityLevel.INFORMATION == "Information" + assert VerbosityLevel.WARNING == "Warning" + assert VerbosityLevel.ERROR == "Error" + assert VerbosityLevel.CRITICAL == "Critical" + assert VerbosityLevel.OFF == "Off" + + def test_execution_type_string_values(self): + assert ExecutionType.DEBUG == "Debug" + assert ExecutionType.RUNTIME == "Runtime" + + def test_enums_are_strings(self): + assert isinstance(SpanStatus.OK, str) + assert isinstance(SpanSource.CODED_AGENTS, str) + assert isinstance(VerbosityLevel.INFORMATION, str) + assert isinstance(ExecutionType.RUNTIME, str) + + +class TestGuidFieldDefaults: + """Guid-typed env-derived fields must default to None, not "". + + v3 ingest (SpanV3Req) binds OrganizationId/FolderKey/TenantId to Guid + fields; an empty string crashes the serializer (400). When the env vars + are unset the span must omit these (None) rather than send "". + """ + + def test_guid_fields_none_when_env_unset( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + for var in ("UIPATH_ORGANIZATION_ID", "UIPATH_TENANT_ID", "UIPATH_FOLDER_KEY"): + monkeypatch.delenv(var, raising=False) + + span = UiPathSpan(id="s", trace_id="t", name="n", attributes={}) + + assert span.organization_id is None + assert span.tenant_id is None + assert span.folder_key is None + + def test_empty_string_env_coerced_to_none( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + for var in ("UIPATH_ORGANIZATION_ID", "UIPATH_TENANT_ID", "UIPATH_FOLDER_KEY"): + monkeypatch.setenv(var, "") + + span = UiPathSpan(id="s", trace_id="t", name="n", attributes={}) + + assert span.organization_id is None + assert span.tenant_id is None + assert span.folder_key is None + + def test_none_guid_fields_omitted_from_to_dict( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Unset Guid fields are omitted from the wire payload, not sent as null.""" + for var in ("UIPATH_ORGANIZATION_ID", "UIPATH_TENANT_ID", "UIPATH_FOLDER_KEY"): + monkeypatch.delenv(var, raising=False) + + d = UiPathSpan(id="s", trace_id="t", name="n", attributes={}).to_dict() + + assert "OrganizationId" not in d + assert "TenantId" not in d + assert "FolderKey" not in d + + def test_set_guid_fields_present_in_to_dict( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Guid fields that are set are still emitted.""" + monkeypatch.setenv("UIPATH_ORGANIZATION_ID", "org-1") + monkeypatch.setenv("UIPATH_FOLDER_KEY", "folder-1") + monkeypatch.delenv("UIPATH_TENANT_ID", raising=False) + + d = UiPathSpan(id="s", trace_id="t", name="n", attributes={}).to_dict() + + assert d["OrganizationId"] == "org-1" + assert d["FolderKey"] == "folder-1" + assert "TenantId" not in d + + +def _make_otel_span(attributes: dict[str, object]) -> Mock: + """Build a minimal mocked OTEL span carrying the given attributes.""" + mock_span = Mock(spec=OTelSpan) + mock_span.get_span_context.return_value = SpanContext( + trace_id=0x123456789ABCDEF0123456789ABCDEF0, + span_id=0x0123456789ABCDEF, + is_remote=False, + ) + mock_span.name = "test-span" + mock_span.parent = None + mock_span.status.status_code = StatusCode.OK + mock_span.attributes = attributes + mock_span.events = [] + mock_span.links = [] + now_ns = int(datetime.now().timestamp() * 1e9) + mock_span.start_time = now_ns + mock_span.end_time = now_ns + 1_000_000 + return mock_span + + +class TestOTelToUiPathSpan: + """OTEL attribute -> top-level UiPathSpan field mapping. + + `_SpanUtils.otel_span_to_uipath_span` lifts a small set of OTEL + span attributes onto dedicated `UiPathSpan` fields surfaced under + `to_dict()`. This test documents that mapping — adding a new row + means the attribute is newly mapped, removing one breaks + downstream consumers. + """ + + ATTRIBUTE_FIELD_MAP = [ + # (otel_attr, span_field, top_level_key, otel_input_int, expected_output) + ("executionType", "execution_type", "ExecutionType", 1, ExecutionType.RUNTIME), + ("agentVersion", "agent_version", "ReferenceVersion", "1.2.3", "1.2.3"), + ("agentId", "reference_id", "ReferenceId", "ref-abc", "ref-abc"), + ("verbosityLevel", "verbosity_level", "VerbosityLevel", 6, VerbosityLevel.OFF), + ] + + @patch.dict(os.environ, {"UIPATH_ORGANIZATION_ID": "test-org"}) + def test_attributes_map_to_top_level_fields(self) -> None: + attrs = { + otel_attr: otel_input + for otel_attr, _, _, otel_input, _ in self.ATTRIBUTE_FIELD_MAP + } + + mock_span = Mock(spec=OTelSpan) + mock_context = SpanContext( + trace_id=0x123456789ABCDEF0123456789ABCDEF0, + span_id=0x0123456789ABCDEF, + is_remote=False, + ) + mock_span.get_span_context.return_value = mock_context + mock_span.name = "test-span" + mock_span.parent = None + mock_span.status.status_code = StatusCode.OK + mock_span.attributes = attrs + mock_span.events = [] + mock_span.links = [] + now_ns = int(datetime.now().timestamp() * 1e9) + mock_span.start_time = now_ns + mock_span.end_time = now_ns + 1_000_000 + + uipath_span = _SpanUtils.otel_span_to_uipath_span(mock_span) + span_dict = uipath_span.to_dict() + + for ( + _, + span_field, + top_level_key, + _, + expected_output, + ) in self.ATTRIBUTE_FIELD_MAP: + assert getattr(uipath_span, span_field) == expected_output, span_field + assert span_dict[top_level_key] == expected_output, top_level_key + + @patch.dict(os.environ, {"UIPATH_ORGANIZATION_ID": "test-org"}) + def test_verbosity_level_omitted_when_unset(self) -> None: + """Spans that don't set verbosityLevel must not carry the key on the wire. + + Backwards compat: pre-existing spans never emitted VerbosityLevel; the + LLMOps backend applies its own default. Adding `"VerbosityLevel": null` + unconditionally would change the wire format for every existing span. + """ + mock_span = Mock(spec=OTelSpan) + mock_context = SpanContext( + trace_id=0x123456789ABCDEF0123456789ABCDEF0, + span_id=0x0123456789ABCDEF, + is_remote=False, + ) + mock_span.get_span_context.return_value = mock_context + mock_span.name = "legacy-span" + mock_span.parent = None + mock_span.status.status_code = StatusCode.OK + mock_span.attributes = {"someOtherAttr": "value"} + mock_span.events = [] + mock_span.links = [] + now_ns = int(datetime.now().timestamp() * 1e9) + mock_span.start_time = now_ns + mock_span.end_time = now_ns + 1_000_000 + + uipath_span = _SpanUtils.otel_span_to_uipath_span(mock_span) + span_dict = uipath_span.to_dict() + + assert uipath_span.verbosity_level is None + assert "VerbosityLevel" not in span_dict + + @patch.dict(os.environ, {"UIPATH_ORGANIZATION_ID": "test-org"}) + def test_verbosity_string_value_maps_to_top_level(self) -> None: + """v3 producers emit verbosityLevel as the StrEnum value "Off" (string). + + The converter must promote it to the top-level VerbosityLevel field so + the LLMOps server can apply its verbosity-Off filter. Before the fix the + string was dropped, the field omitted, and the server defaulted the span + to Information (2) — leaking the AgentDefinition span into the trace. + """ + mock_span = _make_otel_span({"verbosityLevel": "Off"}) + + uipath_span = _SpanUtils.otel_span_to_uipath_span(mock_span) + span_dict = uipath_span.to_dict() + + assert uipath_span.verbosity_level == VerbosityLevel.OFF + assert span_dict["VerbosityLevel"] == VerbosityLevel.OFF + + @pytest.mark.parametrize( + "raw, expected", + [ + (6, VerbosityLevel.OFF), # legacy int + ("Off", VerbosityLevel.OFF), # v3 string value + (2, VerbosityLevel.INFORMATION), # legacy int + ("Information", VerbosityLevel.INFORMATION), # v3 string value + ("Nope", None), # unknown string -> None (server default applies) + ], + ) + @patch.dict(os.environ, {"UIPATH_ORGANIZATION_ID": "test-org"}) + def test_verbosity_accepts_int_and_string(self, raw, expected) -> None: + uipath_span = _SpanUtils.otel_span_to_uipath_span( + _make_otel_span({"verbosityLevel": raw}) + ) + assert uipath_span.verbosity_level == expected + if expected is None: + assert "VerbosityLevel" not in uipath_span.to_dict() + else: + assert uipath_span.to_dict()["VerbosityLevel"] == expected + + @pytest.mark.parametrize( + "raw, expected", + [ + (1, ExecutionType.RUNTIME), # legacy int + ("Runtime", ExecutionType.RUNTIME), # v3 string value + (0, ExecutionType.DEBUG), # legacy int + ("Debug", ExecutionType.DEBUG), # v3 string value + ], + ) + @patch.dict(os.environ, {"UIPATH_ORGANIZATION_ID": "test-org"}) + def test_execution_type_accepts_int_and_string(self, raw, expected) -> None: + uipath_span = _SpanUtils.otel_span_to_uipath_span( + _make_otel_span({"executionType": raw}) + ) + assert uipath_span.execution_type == expected + assert uipath_span.to_dict()["ExecutionType"] == expected + + @pytest.mark.parametrize( + "raw, expected", + [ + (1, SpanSource.AGENTS), # legacy int + ("Agents", SpanSource.AGENTS), # v3 string value + (10, SpanSource.CODED_AGENTS), # legacy int + ("CodedAgents", SpanSource.CODED_AGENTS), # v3 string value + ], + ) + @patch.dict(os.environ, {"UIPATH_ORGANIZATION_ID": "test-org"}) + def test_source_accepts_int_and_string(self, raw, expected) -> None: + uipath_span = _SpanUtils.otel_span_to_uipath_span( + _make_otel_span({"uipath.source": raw}) + ) + assert uipath_span.source == expected + assert uipath_span.to_dict()["Source"] == expected + + +class TestReferenceIdResolution: + """`reference_id` resolution chain. + + `reference_id` is derived from the span's resolved `agentId` attribute + (which itself goes through `resolve_project_id()`), falling back to the + `referenceId` attribute. Falsy values (missing / empty string) at each step + fall through to the next source. The `referenceId` fallback exists for + backwards compatibility with older producers that only emit that attribute. + """ + + @pytest.mark.parametrize( + ("env_value", "attributes", "expected"), + [ + pytest.param( + "env-agent", + {"agentId": "attr-agent", "referenceId": "attr-ref"}, + "env-agent", + id="env-var-overrides-attr", + ), + pytest.param( + None, + {"agentId": "attr-agent", "referenceId": "attr-ref"}, + "attr-agent", + id="agent-id-attr-when-env-unset", + ), + pytest.param( + None, + {"referenceId": "attr-ref"}, + "attr-ref", + id="reference-id-fallback-when-agent-id-missing", + ), + pytest.param( + None, + {"agentId": "", "referenceId": "attr-ref"}, + "attr-ref", + id="reference-id-fallback-when-agent-id-empty", + ), + pytest.param( + None, + {}, + None, + id="none-when-all-sources-missing", + ), + ], + ) + def test_reference_id_chain( + self, + env_value: str | None, + attributes: dict[str, object], + expected: str | None, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + from uipath.platform.common._span_utils import _read_config_id + + _read_config_id.cache_clear() + monkeypatch.delenv(ENV_UIPATH_AGENT_ID, raising=False) + monkeypatch.delenv(ENV_UIPATH_PROJECT_ID, raising=False) + if env_value is None: + monkeypatch.delenv(ENV_PROJECT_KEY, raising=False) + else: + monkeypatch.setenv(ENV_PROJECT_KEY, env_value) + + mock_span = Mock(spec=OTelSpan) + mock_context = SpanContext( + trace_id=0x123456789ABCDEF0123456789ABCDEF0, + span_id=0x0123456789ABCDEF, + is_remote=False, + ) + mock_span.get_span_context.return_value = mock_context + mock_span.name = "test-span" + mock_span.parent = None + mock_span.status.status_code = StatusCode.OK + mock_span.attributes = attributes + mock_span.events = [] + mock_span.links = [] + now_ns = int(datetime.now().timestamp() * 1e9) + mock_span.start_time = now_ns + mock_span.end_time = now_ns + 1_000_000 + + uipath_span = _SpanUtils.otel_span_to_uipath_span(mock_span) + assert uipath_span.reference_id == expected + + +class TestAgentIdResolution: + """`agentId` span attribute resolution via `resolve_project_id()`. + + Priority: `uipath.json#id` (cached, read once per process) > `UIPATH_AGENT_ID` + / `UIPATH_PROJECT_ID` > the legacy `PROJECT_KEY` env var injected by the + executor at runtime. When no source is present the `agentId` attribute is + omitted entirely. + """ + + @staticmethod + def _make_span() -> Mock: + mock_span = Mock(spec=OTelSpan) + mock_context = SpanContext( + trace_id=0x123456789ABCDEF0123456789ABCDEF0, + span_id=0x0123456789ABCDEF, + is_remote=False, + ) + mock_span.get_span_context.return_value = mock_context + mock_span.name = "test-span" + mock_span.parent = None + mock_span.status.status_code = StatusCode.OK + mock_span.attributes = {} + mock_span.events = [] + mock_span.links = [] + now_ns = int(datetime.now().timestamp() * 1e9) + mock_span.start_time = now_ns + mock_span.end_time = now_ns + 1_000_000 + return mock_span + + @staticmethod + def _resolve(monkeypatch: pytest.MonkeyPatch, tmp_path) -> object: + from uipath.platform.common._span_utils import _read_config_id + + _read_config_id.cache_clear() + monkeypatch.delenv("UIPATH_CONFIG_PATH", raising=False) + monkeypatch.delenv(ENV_UIPATH_AGENT_ID, raising=False) + monkeypatch.delenv(ENV_UIPATH_PROJECT_ID, raising=False) + monkeypatch.chdir(tmp_path) + uipath_span = _SpanUtils.otel_span_to_uipath_span( + TestAgentIdResolution._make_span(), serialize_attributes=False + ) + attributes = uipath_span.attributes + assert isinstance(attributes, dict) + return attributes.get("agentId") + + def test_agent_id_from_uipath_json_wins_over_env( + self, monkeypatch: pytest.MonkeyPatch, tmp_path + ) -> None: + (tmp_path / "uipath.json").write_text( + json.dumps({"id": "00000000-0000-0000-0000-000000000001"}) + ) + monkeypatch.setenv(ENV_PROJECT_KEY, "from-env") + assert ( + self._resolve(monkeypatch, tmp_path) + == "00000000-0000-0000-0000-000000000001" + ) + + def test_agent_id_falls_back_to_project_key( + self, monkeypatch: pytest.MonkeyPatch, tmp_path + ) -> None: + # No uipath.json on disk. + monkeypatch.setenv(ENV_PROJECT_KEY, "from-env") + assert self._resolve(monkeypatch, tmp_path) == "from-env" + + def test_agent_id_falls_back_when_config_has_no_id( + self, monkeypatch: pytest.MonkeyPatch, tmp_path + ) -> None: + (tmp_path / "uipath.json").write_text(json.dumps({"functions": {}})) + monkeypatch.setenv(ENV_PROJECT_KEY, "from-env") + assert self._resolve(monkeypatch, tmp_path) == "from-env" + + def test_agent_id_absent_when_no_source( + self, monkeypatch: pytest.MonkeyPatch, tmp_path + ) -> None: + monkeypatch.delenv(ENV_PROJECT_KEY, raising=False) + assert self._resolve(monkeypatch, tmp_path) is None + + def test_non_guid_config_id_is_ignored_and_falls_back( + self, monkeypatch: pytest.MonkeyPatch, tmp_path + ) -> None: + # A malformed (non-GUID) id must not reach ReferenceId; fall back to env. + (tmp_path / "uipath.json").write_text(json.dumps({"id": "not-a-guid"})) + monkeypatch.setenv(ENV_PROJECT_KEY, "from-env") + assert self._resolve(monkeypatch, tmp_path) == "from-env" + + def test_config_id_is_cached( + self, monkeypatch: pytest.MonkeyPatch, tmp_path + ) -> None: + from uipath.platform.common._span_utils import _read_config_id + + first = "00000000-0000-0000-0000-000000000001" + second = "00000000-0000-0000-0000-000000000002" + + _read_config_id.cache_clear() + monkeypatch.delenv("UIPATH_CONFIG_PATH", raising=False) + monkeypatch.chdir(tmp_path) + config = tmp_path / "uipath.json" + + config.write_text(json.dumps({"id": first})) + assert _read_config_id() == first + + # A later edit is not observed: the value is read once and cached. + config.write_text(json.dumps({"id": second})) + assert _read_config_id() == first + + _read_config_id.cache_clear() + assert _read_config_id() == second class TestNormalizeIds: @@ -104,7 +604,7 @@ def test_otel_span_to_uipath_span(self): # Verify the conversion assert isinstance(uipath_span, UiPathSpan) assert uipath_span.name == "test-span" - assert uipath_span.status == 1 # OK + assert uipath_span.status == SpanStatus.OK assert uipath_span.span_type == "CustomSpanType" # Verify IDs are in OTEL hex format @@ -126,7 +626,7 @@ def test_otel_span_to_uipath_span(self): mock_span.status.description = "Test error description" mock_span.status.status_code = StatusCode.ERROR uipath_span = _SpanUtils.otel_span_to_uipath_span(mock_span) - assert uipath_span.status == 2 # Error + assert uipath_span.status == SpanStatus.ERROR @patch.dict( os.environ, @@ -278,12 +778,12 @@ def test_uipath_span_includes_execution_type(self): uipath_span = _SpanUtils.otel_span_to_uipath_span(mock_span) span_dict = uipath_span.to_dict() - assert span_dict["ExecutionType"] == 0 - assert uipath_span.execution_type == 0 + assert span_dict["ExecutionType"] == ExecutionType.DEBUG + assert uipath_span.execution_type == ExecutionType.DEBUG @patch.dict(os.environ, {"UIPATH_ORGANIZATION_ID": "test-org"}) def test_uipath_span_includes_agent_version(self): - """Test that agentVersion from attributes becomes top-level AgentVersion.""" + """Test that agentVersion from attributes becomes top-level ReferenceVersion.""" mock_span = Mock(spec=OTelSpan) trace_id = 0x123456789ABCDEF0123456789ABCDEF0 @@ -305,7 +805,7 @@ def test_uipath_span_includes_agent_version(self): uipath_span = _SpanUtils.otel_span_to_uipath_span(mock_span) span_dict = uipath_span.to_dict() - assert span_dict["AgentVersion"] == "2.0.0" + assert span_dict["ReferenceVersion"] == "2.0.0" assert uipath_span.agent_version == "2.0.0" @patch.dict(os.environ, {"UIPATH_ORGANIZATION_ID": "test-org"}) @@ -332,8 +832,8 @@ def test_uipath_span_execution_type_and_agent_version_both(self): uipath_span = _SpanUtils.otel_span_to_uipath_span(mock_span) span_dict = uipath_span.to_dict() - assert span_dict["ExecutionType"] == 1 - assert span_dict["AgentVersion"] == "1.0.0" + assert span_dict["ExecutionType"] == ExecutionType.RUNTIME + assert span_dict["ReferenceVersion"] == "1.0.0" @patch.dict(os.environ, {"UIPATH_ORGANIZATION_ID": "test-org"}) def test_uipath_span_missing_execution_type_and_agent_version(self): @@ -360,11 +860,11 @@ def test_uipath_span_missing_execution_type_and_agent_version(self): span_dict = uipath_span.to_dict() assert span_dict["ExecutionType"] is None - assert span_dict["AgentVersion"] is None + assert span_dict["ReferenceVersion"] is None @patch.dict(os.environ, {"UIPATH_ORGANIZATION_ID": "test-org"}) - def test_uipath_span_source_defaults_to_robots(self): - """Test that Source defaults to 4 (Robots) and ignores attributes.source.""" + def test_uipath_span_source_defaults_to_coded_agents(self): + """Test that Source defaults to CodedAgents and ignores attributes.source.""" mock_span = Mock(spec=OTelSpan) trace_id = 0x123456789ABCDEF0123456789ABCDEF0 @@ -387,9 +887,9 @@ def test_uipath_span_source_defaults_to_robots(self): uipath_span = _SpanUtils.otel_span_to_uipath_span(mock_span) span_dict = uipath_span.to_dict() - # Top-level Source should be 4 (Robots), string "runtime" is ignored - assert uipath_span.source == 4 - assert span_dict["Source"] == 4 + # Top-level Source should be CodedAgents, string "runtime" is ignored + assert uipath_span.source == SpanSource.CODED_AGENTS + assert span_dict["Source"] == "CodedAgents" # attributes.source string should still be in Attributes JSON attrs = json.loads(span_dict["Attributes"]) @@ -408,7 +908,7 @@ def test_uipath_span_source_override_with_uipath_source(self): mock_span.name = "test-span" mock_span.parent = None mock_span.status.status_code = StatusCode.OK - # uipath.source=1 (Agents) overrides default of 4 (Robots) + # uipath.source=1 (Agents) overrides default of 10 (CodedAgents) mock_span.attributes = {"uipath.source": 1, "source": "runtime"} mock_span.events = [] mock_span.links = [] @@ -421,9 +921,327 @@ def test_uipath_span_source_override_with_uipath_source(self): span_dict = uipath_span.to_dict() # uipath.source overrides - low-code agents use 1 (Agents) - assert uipath_span.source == 1 - assert span_dict["Source"] == 1 + assert uipath_span.source == SpanSource.AGENTS + assert span_dict["Source"] == "Agents" # String source still in Attributes JSON attrs = json.loads(span_dict["Attributes"]) assert attrs["source"] == "runtime" + + @pytest.mark.parametrize(("source_int", "expected"), list(_SOURCE_BY_INT.items())) + @patch.dict(os.environ, {"UIPATH_ORGANIZATION_ID": "test-org"}) + def test_uipath_source_int_maps_to_full_source_enum( + self, source_int: int, expected: SpanSource + ) -> None: + """Every server-known SourceEnum int round-trips (no silent relabeling).""" + mock_span = Mock(spec=OTelSpan) + mock_context = SpanContext( + trace_id=0x123456789ABCDEF0123456789ABCDEF0, + span_id=0x0123456789ABCDEF, + is_remote=False, + ) + mock_span.get_span_context.return_value = mock_context + mock_span.name = "test-span" + mock_span.parent = None + mock_span.status.status_code = StatusCode.OK + mock_span.attributes = {"uipath.source": source_int} + mock_span.events = [] + mock_span.links = [] + now_ns = int(datetime.now().timestamp() * 1e9) + mock_span.start_time = now_ns + mock_span.end_time = now_ns + 1_000_000 + + uipath_span = _SpanUtils.otel_span_to_uipath_span(mock_span) + + assert uipath_span.source == expected + assert uipath_span.to_dict()["Source"] == expected.value + + @patch.dict(os.environ, {"UIPATH_ORGANIZATION_ID": "test-org"}) + def test_unknown_uipath_source_int_warns_and_defaults( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """An unmapped uipath.source int is relabeled CodedAgents and logged.""" + mock_span = Mock(spec=OTelSpan) + mock_context = SpanContext( + trace_id=0x123456789ABCDEF0123456789ABCDEF0, + span_id=0x0123456789ABCDEF, + is_remote=False, + ) + mock_span.get_span_context.return_value = mock_context + mock_span.name = "test-span" + mock_span.parent = None + mock_span.status.status_code = StatusCode.OK + mock_span.attributes = {"uipath.source": 999} + mock_span.events = [] + mock_span.links = [] + now_ns = int(datetime.now().timestamp() * 1e9) + mock_span.start_time = now_ns + mock_span.end_time = now_ns + 1_000_000 + + with caplog.at_level(logging.WARNING): + uipath_span = _SpanUtils.otel_span_to_uipath_span(mock_span) + + assert uipath_span.source == SpanSource.CODED_AGENTS + assert any("999" in record.message for record in caplog.records) + + @patch.dict(os.environ, {"UIPATH_ORGANIZATION_ID": "test-org"}) + def test_bool_attributes_do_not_map_as_ints(self) -> None: + """bool is an int subclass; True must not map to the value-1 enum member.""" + mock_span = Mock(spec=OTelSpan) + mock_context = SpanContext( + trace_id=0x123456789ABCDEF0123456789ABCDEF0, + span_id=0x0123456789ABCDEF, + is_remote=False, + ) + mock_span.get_span_context.return_value = mock_context + mock_span.name = "test-span" + mock_span.parent = None + mock_span.status.status_code = StatusCode.OK + mock_span.attributes = { + "executionType": True, + "verbosityLevel": True, + "uipath.source": True, + } + mock_span.events = [] + mock_span.links = [] + now_ns = int(datetime.now().timestamp() * 1e9) + mock_span.start_time = now_ns + mock_span.end_time = now_ns + 1_000_000 + + uipath_span = _SpanUtils.otel_span_to_uipath_span(mock_span) + + assert uipath_span.execution_type is None + assert uipath_span.verbosity_level is None + assert uipath_span.source == SpanSource.CODED_AGENTS + + +class TestUiPathSpanDictUsesStrings: + def test_default_status_is_ok_string(self): + span = UiPathSpan( + id="a" * 16, + trace_id="b" * 32, + name="test", + attributes={}, + ) + d = span.to_dict() + assert d["Status"] == "Ok" + + def test_default_source_is_coded_agents_string(self): + span = UiPathSpan( + id="a" * 16, + trace_id="b" * 32, + name="test", + attributes={}, + ) + d = span.to_dict() + assert d["Source"] == "CodedAgents" + + def test_verbosity_level_serializes_as_string(self): + span = UiPathSpan( + id="a" * 16, + trace_id="b" * 32, + name="test", + attributes={}, + verbosity_level=VerbosityLevel.OFF, + ) + d = span.to_dict() + assert d["VerbosityLevel"] == "Off" + + def test_execution_type_serializes_as_string(self): + span = UiPathSpan( + id="a" * 16, + trace_id="b" * 32, + name="test", + attributes={}, + execution_type=ExecutionType.RUNTIME, + ) + d = span.to_dict() + assert d["ExecutionType"] == "Runtime" + + +class TestOtelSpanConversionUsesStrEnums: + def _make_mock_span(self, status_code=StatusCode.OK, attributes=None): + from datetime import datetime + from unittest.mock import Mock + + from opentelemetry.trace import SpanContext + + mock_span = Mock() + mock_context = SpanContext( + trace_id=0x123456789ABCDEF0123456789ABCDEF0, + span_id=0x0123456789ABCDEF, + is_remote=False, + ) + mock_span.get_span_context.return_value = mock_context + mock_span.name = "test-span" + mock_span.parent = None + mock_span.status.status_code = status_code + mock_span.status.description = None + mock_span.attributes = attributes or {} + mock_span.events = [] + mock_span.links = [] + now_ns = int(datetime.now().timestamp() * 1e9) + mock_span.start_time = now_ns + mock_span.end_time = now_ns + 1_000_000 + return mock_span + + @patch.dict(os.environ, {"UIPATH_ORGANIZATION_ID": "test-org"}) + def test_ok_status_maps_to_str_enum(self): + span = _SpanUtils.otel_span_to_uipath_span(self._make_mock_span()) + assert span.status == SpanStatus.OK + assert span.to_dict()["Status"] == "Ok" + + @patch.dict(os.environ, {"UIPATH_ORGANIZATION_ID": "test-org"}) + def test_error_status_maps_to_str_enum(self): + mock_span = self._make_mock_span(status_code=StatusCode.ERROR) + mock_span.status.description = "something went wrong" + span = _SpanUtils.otel_span_to_uipath_span(mock_span) + assert span.status == SpanStatus.ERROR + assert span.to_dict()["Status"] == "Error" + + @patch.dict(os.environ, {"UIPATH_ORGANIZATION_ID": "test-org"}) + def test_default_source_is_coded_agents(self): + span = _SpanUtils.otel_span_to_uipath_span(self._make_mock_span()) + assert span.source == SpanSource.CODED_AGENTS + assert span.to_dict()["Source"] == "CodedAgents" + + @patch.dict(os.environ, {"UIPATH_ORGANIZATION_ID": "test-org"}) + def test_execution_type_int_maps_to_str_enum(self): + mock_span = self._make_mock_span(attributes={"executionType": 1}) + span = _SpanUtils.otel_span_to_uipath_span(mock_span) + assert span.execution_type == ExecutionType.RUNTIME + assert span.to_dict()["ExecutionType"] == "Runtime" + + @patch.dict(os.environ, {"UIPATH_ORGANIZATION_ID": "test-org"}) + def test_verbosity_level_int_maps_to_str_enum(self): + mock_span = self._make_mock_span(attributes={"verbosityLevel": 6}) + span = _SpanUtils.otel_span_to_uipath_span(mock_span) + assert span.verbosity_level == VerbosityLevel.OFF + assert span.to_dict()["VerbosityLevel"] == "Off" + + +# --------------------------------------------------------------------------- +# ReferenceHierarchySpanProcessor +# --------------------------------------------------------------------------- + + +class TestReferenceHierarchySpanProcessor: + """Tests for ReferenceHierarchySpanProcessor.on_start. + + The only change made to this class was aligning the parent_context + parameter type annotation with the SpanProcessor base class + (Optional[context_api.Context] instead of context_api.Context | None). + These tests verify the processor is a proper SpanProcessor subclass and + that on_start behaves correctly under all valid call forms. + """ + + def setup_method(self) -> None: + token = ReferenceContextAccessor.set(None) + ReferenceContextAccessor.reset(token) + + def _make_mock_span(self) -> Mock: + mock = Mock(spec=OTelSpan) + return mock + + def test_is_span_processor_subclass(self) -> None: + assert issubclass(ReferenceHierarchySpanProcessor, SpanProcessor) + + def test_on_start_with_default_parent_context(self) -> None: + ref_ctx = ReferenceContext.Empty.add( + "agent", "550e8400-e29b-41d4-a716-446655440001" + ) + token = ReferenceContextAccessor.set(ref_ctx) + try: + processor = ReferenceHierarchySpanProcessor() + mock_span = self._make_mock_span() + processor.on_start(mock_span) # parent_context omitted — uses default None + mock_span.set_attribute.assert_called_once() + key, value = mock_span.set_attribute.call_args[0] + assert key == "uipath.reference_hierarchy" + assert json.loads(value)[0]["serviceType"] == "agent" + finally: + ReferenceContextAccessor.reset(token) + + def test_on_start_with_explicit_none_parent_context(self) -> None: + ref_ctx = ReferenceContext.Empty.add( + "maestro", "550e8400-e29b-41d4-a716-446655440010" + ) + token = ReferenceContextAccessor.set(ref_ctx) + try: + processor = ReferenceHierarchySpanProcessor() + mock_span = self._make_mock_span() + processor.on_start(mock_span, parent_context=None) + mock_span.set_attribute.assert_called_once() + finally: + ReferenceContextAccessor.reset(token) + + def test_on_start_with_real_context_object(self) -> None: + ref_ctx = ReferenceContext.Empty.add( + "agent", "550e8400-e29b-41d4-a716-446655440001" + ) + token = ReferenceContextAccessor.set(ref_ctx) + try: + processor = ReferenceHierarchySpanProcessor() + mock_span = self._make_mock_span() + otel_ctx = context_api.create_key("test") + real_ctx = context_api.set_value(otel_ctx, "val") + processor.on_start(mock_span, parent_context=real_ctx) + mock_span.set_attribute.assert_called_once() + finally: + ReferenceContextAccessor.reset(token) + + def test_on_start_noop_when_no_reference_context(self) -> None: + processor = ReferenceHierarchySpanProcessor() + mock_span = self._make_mock_span() + processor.on_start(mock_span) + mock_span.set_attribute.assert_not_called() + + def test_on_start_stamps_full_hierarchy(self) -> None: + ref_ctx = ReferenceContext.Empty.add( + "maestro", "550e8400-e29b-41d4-a716-446655440010", "2.0" + ).add("agent", "550e8400-e29b-41d4-a716-446655440011") + token = ReferenceContextAccessor.set(ref_ctx) + try: + processor = ReferenceHierarchySpanProcessor() + mock_span = self._make_mock_span() + processor.on_start(mock_span) + key, value = mock_span.set_attribute.call_args[0] + hierarchy = json.loads(value) + assert len(hierarchy) == 2 + assert hierarchy[0]["serviceType"] == "maestro" + assert hierarchy[0]["version"] == "2.0" + assert hierarchy[1]["serviceType"] == "agent" + assert "version" not in hierarchy[1] + finally: + ReferenceContextAccessor.reset(token) + + +class TestLiveSpanEndTime: + """A live (not-yet-ended) OTEL span must convert with end_time=None. + + Fabricating EndTime=now() for in-progress upserts (the RUNNING -> OK + lifecycle used by upsert_span) makes open snapshots indistinguishable + from ended spans downstream: the traceview UI shows phantom sub-ms + durations and the Insights OTLP export's unclosed-span filter never + fires, so every span exports twice. + """ + + @patch.dict(os.environ, {"UIPATH_ORGANIZATION_ID": "test-org"}) + def test_live_span_converts_with_none_end_time(self) -> None: + mock_span = _make_otel_span({}) + mock_span.end_time = None + + uipath_span = _SpanUtils.otel_span_to_uipath_span(mock_span) + span_dict = uipath_span.to_dict() + + assert uipath_span.end_time is None + assert span_dict["EndTime"] is None + + @patch.dict(os.environ, {"UIPATH_ORGANIZATION_ID": "test-org"}) + def test_ended_span_keeps_real_end_time(self) -> None: + mock_span = _make_otel_span({}) + + uipath_span = _SpanUtils.otel_span_to_uipath_span(mock_span) + + assert uipath_span.end_time is not None + assert uipath_span.to_dict()["EndTime"] == uipath_span.end_time diff --git a/packages/uipath-platform/tests/services/test_uipath_llm_integration.py b/packages/uipath-platform/tests/services/test_uipath_llm_integration.py index 124ccad8b..9e2292c60 100644 --- a/packages/uipath-platform/tests/services/test_uipath_llm_integration.py +++ b/packages/uipath-platform/tests/services/test_uipath_llm_integration.py @@ -7,6 +7,7 @@ from uipath.platform.chat import ( AutoToolChoice, ChatModels, + RequiredToolChoice, SpecificToolChoice, ToolDefinition, ToolFunctionDefinition, @@ -369,6 +370,87 @@ async def test_tool_call_required_mocked(self, mock_request, llm_service): assert result.choices[0].message.tool_calls[0].arguments["name"] == "John" assert result.choices[0].message.tool_calls[0].arguments["password"] == "1234" + @pytest.mark.asyncio + @patch.object(UiPathLlmChatService, "request_async") + async def test_raw_dict_tool_passthrough_mocked(self, mock_request, llm_service): + """A tool supplied as a raw dict is sent unchanged, preserving nested schema. + + ToolDefinition's converter only emits flat properties, so callers that need + an arbitrary nested JSON schema (e.g. the eval mockers) pass the tool as a + dict already in UiPath wire format. It must reach the gateway verbatim. + """ + mock_response = MagicMock() + mock_response.json.return_value = { + "id": "chatcmpl-raw", + "object": "chat.completion", + "created": 1677858242, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_raw", + "name": "submit_tool_response", + "arguments": {"response": {"items": [{"sku": "A1"}]}}, + } + ], + }, + "finish_reason": "tool_calls", + } + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + "cache_read_input_tokens": None, + }, + } + mock_request.return_value = mock_response + + nested_tool = { + "name": "submit_tool_response", + "description": "Return the simulated response matching the schema.", + "parameters": { + "type": "object", + "properties": { + "response": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": {"sku": {"type": "string"}}, + }, + } + }, + } + }, + "required": ["response"], + }, + } + + result = await llm_service.chat_completions( + messages=[{"role": "user", "content": "go"}], + model=ChatModels.gpt_4_1_mini_2025_04_14, + tools=[nested_tool], + tool_choice=RequiredToolChoice(), + ) + + mock_request.assert_called_once() + _, kwargs = mock_request.call_args + body = kwargs["json"] + # The dict tool is forwarded byte-for-byte, nested array schema intact. + assert body["tools"] == [nested_tool] + assert body["tool_choice"] == {"type": "required"} + assert result.choices[0].message.tool_calls[0].arguments == { + "response": {"items": [{"sku": "A1"}]} + } + @pytest.mark.asyncio @patch.object(UiPathLlmChatService, "request_async") async def test_chat_with_conversation_history_mocked( diff --git a/packages/uipath-platform/uv.lock b/packages/uipath-platform/uv.lock index 0ea4a2f63..2e506b0af 100644 --- a/packages/uipath-platform/uv.lock +++ b/packages/uipath-platform/uv.lock @@ -2,6 +2,13 @@ version = 1 revision = 3 requires-python = ">=3.11" +[options] +exclude-newer = "2026-07-29T07:23:36.9681123Z" +exclude-newer-span = "P2D" + +[options.exclude-newer-package] +uipath-core = false + [[package]] name = "annotated-types" version = "0.7.0" @@ -1056,7 +1063,7 @@ wheels = [ [[package]] name = "uipath-core" -version = "0.5.10" +version = "0.5.31" source = { editable = "../uipath-core" } dependencies = [ { name = "opentelemetry-instrumentation" }, @@ -1088,9 +1095,10 @@ dev = [ [[package]] name = "uipath-platform" -version = "0.1.18" +version = "0.2.17" source = { editable = "." } dependencies = [ + { name = "anyio" }, { name = "httpx" }, { name = "pydantic-function-models" }, { name = "sqlparse" }, @@ -1116,6 +1124,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "anyio", specifier = ">=4.0.0" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "pydantic-function-models", specifier = ">=0.1.11" }, { name = "sqlparse", specifier = ">=0.5.5" }, diff --git a/packages/uipath/CLAUDE.md b/packages/uipath/CLAUDE.md index 9b4c372c2..a5b489af5 100644 --- a/packages/uipath/CLAUDE.md +++ b/packages/uipath/CLAUDE.md @@ -76,7 +76,7 @@ Plugin-based evaluator registration system with deterministic and LLM-based eval - **`evaluators/`** — 11 active evaluators: ExactMatch, Contains, JsonSimilarity, BinaryClassification, MulticlassClassification, LLMJudgeOutput, LLMJudgeTrajectory, ToolCallOrder/Args/Count/Output. Plus ~7 legacy evaluators. - **`mocks/`** — LLM mocking, input mocking, cache management, mockito integration. Key exports: `mockable`, `UiPathMockRuntime`, `MockingContext`. - **`runtime/`** — `UiPathEvalRuntime`, `UiPathEvalContext`, `evaluate()` entry point, parallelization, exporters. -- **`models/`** — `AgentExecution`, `EvaluationResult`, `ToolCall`, `LLMResponse`, evaluator type enums. +- **`models/`** — `WorkloadExecution`, `EvaluationResult`, `ToolCall`, `LLMResponse`, evaluator type enums. ## Functions Module (`src/uipath/functions/`) diff --git a/packages/uipath/docs/FAQ.md b/packages/uipath/docs/FAQ.md index c1cd1aec2..602a42196 100644 --- a/packages/uipath/docs/FAQ.md +++ b/packages/uipath/docs/FAQ.md @@ -1,6 +1,6 @@ # Frequently Asked Questions (FAQ) -### Q: Why am I getting a "Failed to prepare environment" error when deploying my python agent to UiPath Cloud Platform? +### Q: Why am I getting a "Failed to prepare environment" error when deploying my Python project to UiPath Cloud Platform? #### Error Message @@ -30,7 +30,7 @@ #### Description -This error might occur when deploying coded-agents to UiPath Cloud Platform, even though the same project might work correctly in your local environment. The issue is often related to how Python packages are discovered and distributed during the cloud deployment process. +This error might occur when deploying coded functions or coded agents to UiPath Cloud Platform, even though the same project might work correctly in your local environment. The issue is often related to how Python packages are discovered and distributed during the cloud deployment process. #### Common Causes @@ -283,7 +283,7 @@ If you encounter SSL certificate errors: //// -### Q: Why are my agent runs hanging on UiPath Cloud Platform? +### Q: Why are my job runs hanging on UiPath Cloud Platform? #### Error Message @@ -298,7 +298,7 @@ You may see errors like these in the logs panel: #### Description -If your Python agent runs are hanging or not completing when deployed to UiPath Cloud Platform's serverless environment, this may be caused by a library incompatibility issue from an outdated version of the UiPath Python library. +If your Python job runs are hanging or not completing when deployed to UiPath Cloud Platform's serverless environment, this may be caused by a library incompatibility issue from an outdated version of the UiPath Python library. #### Solution diff --git a/packages/uipath/docs/assets/llms.txt b/packages/uipath/docs/assets/llms.txt deleted file mode 100644 index ac9907c58..000000000 --- a/packages/uipath/docs/assets/llms.txt +++ /dev/null @@ -1,58 +0,0 @@ -# UiPath Python SDK Documentation -> https://uipath.github.io/uipath-python/ - -A Python SDK for programmatic interaction with UiPath Cloud Platform services, featuring CLI tools for automation creation, packaging, and deployment. Includes support for LangChain, LlamaIndex, and Model Context Protocol (MCP) agent frameworks. - -## Core Documentation - -### Getting Started -- https://uipath.github.io/uipath-python/ - Main landing page -- https://uipath.github.io/uipath-python/core/getting_started - SDK quickstart guide -- https://uipath.github.io/uipath-python/FAQ - Frequently Asked Questions -- https://uipath.github.io/uipath-python/CONTRIBUTING - Contribution guidelines -- https://uipath.github.io/uipath-python/release_policy - Release policy - -### SDK Features -- https://uipath.github.io/uipath-python/core/processes - Process automation -- https://uipath.github.io/uipath-python/core/jobs - Job management -- https://uipath.github.io/uipath-python/core/assets - Asset storage and retrieval -- https://uipath.github.io/uipath-python/core/queues - Queue operations -- https://uipath.github.io/uipath-python/core/resource_catalog - Resources search -- https://uipath.github.io/uipath-python/core/buckets - Cloud storage buckets -- https://uipath.github.io/uipath-python/core/attachments - File attachments -- https://uipath.github.io/uipath-python/core/actions - Action Center integration -- https://uipath.github.io/uipath-python/core/entities - Data Service integration -- https://uipath.github.io/uipath-python/core/connections - External connections -- https://uipath.github.io/uipath-python/core/documents - Document handling -- https://uipath.github.io/uipath-python/core/documents_models - Document data models -- https://uipath.github.io/uipath-python/core/environment_variables - Environment configuration -- https://uipath.github.io/uipath-python/core/guardrails - Guardrails validation -- https://uipath.github.io/uipath-python/core/traced - Tracing and observability - -### LLM & AI Features -- https://uipath.github.io/uipath-python/core/llm_gateway - LLM Gateway for model access -- https://uipath.github.io/uipath-python/core/context_grounding - RAG and semantic search - -### Agent Frameworks -- https://uipath.github.io/uipath-python/mcp/quick_start - Model Context Protocol (MCP) SDK -- https://uipath.github.io/uipath-python/langchain/quick_start - LangChain integration -- https://uipath.github.io/uipath-python/llamaindex/quick_start - LlamaIndex integration - -### CLI Tools -- https://uipath.github.io/uipath-python/cli/ - Command-line interface reference - -## Supported LLM Models - -The following LLM models are referenced in examples and evaluations throughout the repository: - -### OpenAI Models -- gpt-4o-mini-2024-07-18 - Used in samples and evaluations -- gpt-4o-2024-08-06 - Primary model for agent examples -- gpt-4 - General purpose examples -- gpt-4.1-2025-04-14 - LLM-as-judge evaluator - -### Google Models -- gemini-1.5-flash - Used in Google ADK agent sample - -### Embedding Models -- text-embedding-3-large - Azure OpenAI embeddings for RAG diff --git a/packages/uipath/docs/cli/index.md b/packages/uipath/docs/cli/index.md index deb875353..e74a4d83e 100644 --- a/packages/uipath/docs/cli/index.md +++ b/packages/uipath/docs/cli/index.md @@ -1,5 +1,7 @@ # CLI Reference +The following commands apply to both **coded functions** and **coded agents**. The entry point name (`main`, `agent`, or any key you define in `uipath.json`) is the first argument to `run`, `debug`, `eval`, and `invoke`. + ::: mkdocs-click :module: uipath._cli :command: auth @@ -32,6 +34,37 @@ Select tenant number: 0 Selected tenant: Tenant1 ✓ Authentication successful. ``` + +/// info | Unattended Authentication (Client Credentials) + +For CI/CD pipelines and other non-interactive contexts, authenticate with the OAuth client credentials flow by passing all three of `--client-id`, `--client-secret`, and `--base-url`. The CLI exchanges them for an access token and writes it to the same on-disk session used by interactive logins, so subsequent commands like `uipath publish` and `uipath invoke` work without further setup. + +The `--base-url` must point at the tenant scope (`https:////`). The optional `--scope` flag controls the OAuth scopes requested and defaults to `OR.Execution`. Pass a space-separated list (for example `"OR.Execution OR.Queues"`) to request additional scopes — match the scopes you granted to the External Application and the operations you intend to run. + +**Setup:** + +1. In the Automation Cloud **Admin** page, open **External Applications** and create one of type *Confidential*. Grant it the Orchestrator scopes you need (for example `OR.Execution`). See the [External Applications guide](https://docs.uipath.com/automation-cloud/automation-cloud/latest/admin-guide/managing-external-applications) for details. +2. Copy the generated **App ID** and **App Secret** — these become `--client-id` and `--client-secret`. + +**Example:** + + +```shell +> uipath auth --client-id 12345678-c4c5-4f1f-93ff-4f5ab47d57ea \ + --client-secret 'your-secret' \ + --base-url https://cloud.uipath.com/your-org/your-tenant +✓ Authentication successful. +> uipath publish --tenant +``` + +/// warning +Treat `--client-secret` as a credential. In CI, prefer reading it from a secret store and passing it on the command line, rather than committing it to source control or leaving it in shell history. +/// + +**Configuring the same flow in code:** if you would rather skip the CLI session and pass credentials directly to the SDK, the [`asset-modifier-agent` sample](https://github.com/UiPath/uipath-python/tree/main/packages/uipath/samples/asset-modifier-agent) shows how to construct a `UiPath` client with `client_id`, `client_secret`, `scope`, and `base_url` from environment variables. + +/// + --- ::: mkdocs-click @@ -82,6 +115,20 @@ Running `uipath init` will process these function definitions and create the cor ✓ Created '.uipath/studio_metadata.json' file. ✓ Created: CLAUDE.md, CLI_REFERENCE.md, SDK_REFERENCE.md, AGENTS.md, REQUIRED_STRUCTURE.md. ``` + +/// info +### About the `.mermaid` files + +`uipath init` generates one `.mermaid` file per function/agent containing a static call graph, rendered in the UiPath Orchestrator UI. These files are regenerated on every `uipath init`. +/// + +/// warning +### About the `id` field + +The first `uipath init` mints a stable `id` (GUID) into `uipath.json` and preserves it across subsequent runs. It is what identifies your project consistently wherever it is deployed and run. + +Do not change or remove it. Changing it makes the project look like a brand-new, unrelated one, so you lose the link to everything previously published and tracked under the old id. `uipath pack` rejects an `id` that is not a valid GUID. +/// --- ::: mkdocs-click @@ -95,7 +142,7 @@ For step-by-step debugging with breakpoints and variable inspection (supported f ```console # Install debugpy package uv pip install debugpy -# Run agent with debugging enabled +# Run with debugging enabled uipath run [ENTRYPOINT] [INPUT] --debug ``` For vscode: @@ -117,19 +164,19 @@ Depending on the shell you are using, it may be necessary to escape the input js /// tab | Bash/ZSH ```console -uipath run agent '{"topic": "UiPath"}' +uipath run main '{"message": "hello"}' ``` /// /// tab | Windows CMD ```console -uipath run agent "{""topic"": ""UiPath""}" +uipath run main "{""message"": ""hello""}" ``` /// /// tab | Windows PowerShell ```console -uipath run agent '{\"topic\":\"uipath\"}' +uipath run main '{\"message\":\"hello\"}' ``` /// @@ -161,6 +208,7 @@ By default, the following file types are included in the `.nupkg` file: - `.json` - `.yaml` - `.yml` +- `.md` --- @@ -175,7 +223,7 @@ To include additional files, update the `uipath.json` file by adding a `packOpti "" ], "fileExtensionsIncluded": [ - "" + "" ] } } @@ -197,6 +245,25 @@ authors = [{name = "Your Name", email = "your.email@example.com"}] ``` /// +/// info +### Dependency Locking + +By default, `uipath pack` includes `uv.lock` in the `.nupkg` (creating it if it does not exist). The executor then installs the pinned versions from the lock file, so every run uses the exact same dependency versions. + +Use `--nolock` to opt out — `uv.lock` is not added to the package. With no lock file present, the executor resolves dependencies on each run and picks the latest versions compatible with the constraints in your `pyproject.toml`. + + +```shell +> uipath pack --nolock +⠋ Packaging project ... +✓ Project successfully packaged. +``` + +**When to lock (default):** you want reproducible runs and protection against breaking changes or malicious upgrades in your dependencies. The versions you tested with are the versions that run. + +**When to use `--nolock`:** you want each run to pick up the latest patches automatically within your declared constraints, or your project does not use uv. +/// + ```shell > uipath pack @@ -255,7 +322,7 @@ Selected feed: Orchestrator Tenant Processes Feed ```shell -> uipath invoke agent '{"topic": "UiPath"}' +> uipath invoke main '{"message": "hello"}' ⠴ Loading configuration ... ⠴ Starting job ... ✨ Job started successfully! @@ -283,6 +350,19 @@ Importing referenced resources to Studio Web project... 🔵 Resource import summary: 0 total resources - 0 created, 0 updated, 0 unchanged, 0 not found ``` + +/// info +### Dependency Locking + +By default, `uipath push` includes `uv.lock` in the upload (creating it if it does not exist). The executor then installs the pinned versions from the lock file, so every run uses the exact same dependency versions. + +Use `--nolock` to opt out — `uv.lock` is not uploaded. With no lock file present, the executor resolves dependencies on each run and picks the latest versions compatible with the constraints in your `pyproject.toml`. + +**When to lock (default):** you want reproducible runs and protection against breaking changes or malicious upgrades in your dependencies. The versions you tested with are the versions that run. + +**When to use `--nolock`:** you want each run to pick up the latest patches automatically within your declared constraints, or your project does not use uv. +/// + --- ::: mkdocs-click @@ -302,3 +382,92 @@ Processing: uipath.json File 'uipath.json' is up to date ✓ Project pulled successfully ``` +--- + +::: mkdocs-click + :module: uipath._cli + :command: debug + :depth: 1 + :style: table + +Runs your project under the debug runtime, with a debug bridge attached. Locally, the bridge is the interactive **console** (read commands from stdin, stop at breakpoints). In the cloud, the bridge is **SignalR** (driven by Studio Web / Orchestrator). The `--attach` flag lets you override that default, including `none` for executors that need the debug command's surrounding behavior (bindings fetch, state streaming) but cannot speak the interactive debug protocol. + +### Attach modes + +| Mode | When to use | +|------|-------------| +| `signalr` | Remote runs driven by Studio Web / Orchestrator. Default when `job_id` is set. | +| `console` | Local interactive debugging from the terminal. Default when no `job_id`. | +| `none` | Run under the debug command without attaching a debugger. No wait-for-start gate, no breakpoints, no step mode. | + +/// info +`--attach` selects the **debug bridge**. It's unrelated to `--debug`, which starts a `debugpy` server for Python-level breakpoints in your IDE. The two can be combined. +/// + + + +```shell +> uipath debug main '{"message": "test"}' +Debug Mode Commands + c, continue Continue until next breakpoint + s, step Step to next node + b Set breakpoint at + l, list List all breakpoints + r Remove breakpoint at + h, help Show help + q, quit Exit debugger +▶ START +> b analyze_sentiment +✓ Breakpoint set at: analyze_sentiment +> c +──────────────────────────────────────── +■ BREAKPOINT analyze_sentiment (before) +Next: analyze_sentiment +──────────────────────────────────────── +> s +● analyze_sentiment +> c +✓ Execution completed +``` +--- + +::: mkdocs-click + :module: uipath._cli + :command: eval + :depth: 1 + :style: table + +Runs an evaluation set against your project. Entry point and eval set are auto-discovered from the project if not passed explicitly. Evaluations run in parallel (see `--workers`) and, unless `--no-report` is passed, results are reported back to Studio Web when `UIPATH_PROJECT_ID` is set. + +### Common flags + +| Flag | Purpose | +|------|---------| +| `--eval-ids` | Run only a subset of evaluations by id. | +| `--workers` | Parallel workers for running evaluations (default 1). | +| `--no-report` | Skip reporting results back to UiPath. | +| `--enable-mocker-cache` | Cache LLM mocker responses across runs. | +| `--input-overrides` | Per-eval input overrides, merged into the eval's input. | +| `--trace-file` | Write OpenTelemetry traces to a JSONL file for offline inspection. | +| `--resume` | Resume evaluation from a previous suspended state. | + + + +```shell +> uipath eval +⠋ Running evaluations ... + Weather in Paris + LLM Judge Output 0.7 + Tool Call Arguments 1.0 + Tool Call Count 1.0 + Tool Call Order 1.0 + +Evaluation Results +┏━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┓ +┃ Evaluation ┃ LLM Judge Output ┃ Tool Call Args ┃ Tool Call Count ┃ Tool Call Order ┃ +┡━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━┩ +│ Weather in Paris │ 0.7 │ 1.0 │ 1.0 │ 1.0 │ +├────────────────────┼────────────────────┼────────────────────┼────────────────────┼────────────────────┤ +│ Average │ 0.7 │ 1.0 │ 1.0 │ 1.0 │ +└────────────────────┴────────────────────┴────────────────────┴────────────────────┴────────────────────┘ +``` diff --git a/packages/uipath/docs/core/agents.md b/packages/uipath/docs/core/agents.md new file mode 100644 index 000000000..21b812647 --- /dev/null +++ b/packages/uipath/docs/core/agents.md @@ -0,0 +1,244 @@ +# Python Coded Agents + +A coded agent is Python code that uses an LLM reasoning loop to make decisions, call tools, and produce a result. You write the agent logic using a framework of your choice — the `uipath` SDK provides the platform layer: authentication, assets, buckets, connections, tracing, and human-in-the-loop. Package it with the CLI and deploy it as an Orchestrator job. + +Use a coded agent when your automation needs multi-step reasoning, dynamic tool selection, or LLM-driven decisions. Use a [coded function](./functions.md) when your logic is deterministic and no LLM is required. + +--- + +## Architecture + +Every coded agent is built from two layers: + +| Layer | Package | Responsibility | +|-------|---------|---------------| +| **Platform** | `uipath` | Auth, assets, buckets, connections, tracing, human-in-the-loop, CLI, packaging | +| **Framework** | one extension (see below) | LLM calls, tool routing, agent loop, memory | + +The `uipath` package is always required. Add one framework extension on top: + +| Framework | Package | Best for | +|-----------|---------|---------| +| LangChain / LangGraph | `uipath-langchain` | Graph-based agents, complex multi-step flows | +| LlamaIndex | `uipath-llamaindex` | RAG-heavy agents, document reasoning | +| OpenAI Agents SDK | `uipath-openai-agents` | OpenAI-native tool use, handoffs | +| PydanticAI | `uipath-pydantic-ai` | Type-safe agents with Pydantic models | +| Google ADK | `uipath-google-adk` | Gemini models, Google ecosystem | +| UiPath Agent Framework | `uipath-agent-framework` | UiPath-native agent primitives | + +--- + +## Quickstart + +The example below uses LangChain. Swap `uipath-langchain` for the framework of your choice. + +//// tab | uv + + + +```shell +> mkdir my-agent && cd my-agent +> uv init . --python 3.11 +> uv add uipath uipath-langchain + +> uipath auth +⠋ Authenticating with UiPath ... +✓ Authentication successful. + +> uipath new agent +✓ Created new agent project. + +> uipath init +⠋ Initializing UiPath project ... +✓ Created 'entry-points.json' file. +✓ Created 'bindings.json' file. + +> uipath run agent '{"message": "hello"}' +``` + +//// + +//// tab | pip + + + +```shell +> mkdir my-agent && cd my-agent +> python -m venv .venv +> source .venv/bin/activate +> pip install uipath uipath-langchain + +> uipath auth +⠋ Authenticating with UiPath ... +✓ Authentication successful. + +> uipath new agent +✓ Created new agent project. + +> uipath init +⠋ Initializing UiPath project ... +✓ Created 'entry-points.json' file. +✓ Created 'bindings.json' file. + +> uipath run agent '{"message": "hello"}' +``` + +//// + +--- + +## Project Structure + +``` +my-agent/ +├── main.py # agent graph +├── langgraph.json # graph entry points (framework-specific) +├── pyproject.toml # project metadata and dependencies +├── entry-points.json # generated — I/O JSON Schema +└── bindings.json # generated — resource binding overrides +``` + +### `langgraph.json` + +```json +{ + "graphs": { + "agent": "./main.py:graph" + } +} +``` + +Declares the agent's graph entry points. The filename is framework-specific — `langgraph.json` for LangChain/LangGraph, `llamaindex.json` for LlamaIndex, and so on. Its presence, together with the framework dependency below, is what marks the project as a coded agent. + +### `pyproject.toml` + +```toml +[project] +name = "my-agent" +version = "0.1.0" +description = "..." +authors = [{ name = "Your Name", email = "you@example.com" }] +requires-python = ">=3.11" +dependencies = ["uipath>=2.0", "uipath-langchain>=2.0"] +``` + +Standard metadata plus the framework dependency (`uipath-langchain` here). The framework graph file and this dependency identify the project as a coded agent — `pyproject.toml` needs no UiPath-specific entries, and `uipath.json` carries no agent entry. + +--- + +## Input & Output + +Define `Input` and `Output` the same way as [coded functions](./functions.md#input--output) — a stdlib `@dataclass`, a pydantic `BaseModel`, or `pydantic.dataclasses.dataclass`: + +```python +from dataclasses import dataclass + + +@dataclass +class Input: + message: str + + +@dataclass +class Output: + response: str + + +def agent(input: Input) -> Output: + ... +``` + +--- + +## Platform Services + +The `uipath` SDK gives your agent access to Orchestrator resources at runtime — credentials are injected automatically when running as a job. + +```python +from uipath.platform import UiPath + +sdk = UiPath() +``` + +The full set of Orchestrator services is available to agents: + +- **Assets** — read credentials and configuration: [Assets reference](./assets.md) +- **Buckets** — download and upload files: [Buckets reference](./buckets.md) +- **Connections** — Integration Service connections for ERP and SaaS: [Connections reference](./connections.md) +- **Context Grounding** — semantic search over enterprise data: [Context Grounding reference](./context_grounding.md) + +--- + +## Tracing + +Apply `@traced` to custom steps inside your agent to make them visible in the Orchestrator job trace view and Maestro dashboards. Do **not** trace the entry point — the runtime wraps it automatically. + +```python +from uipath.tracing import traced + + +@traced(name="lookup_vendor", run_type="uipath") +def lookup_vendor(vendor_id: str) -> dict: + ... +``` + +See [Tracing](./traced.md) for the full decorator reference. + +--- + +## Framework Guides + +Each framework extension has its own getting started guide and sample agents: + +| Framework | Guide | Samples | +|-----------|-------|---------| +| LangChain / LangGraph | [Get Started](../langchain/quick_start.md) | [Samples](https://github.com/UiPath/uipath-langchain-python/tree/main/samples) | +| LlamaIndex | [Get Started](../llamaindex/quick_start.md) | [Samples](https://github.com/UiPath/uipath-integrations-python/tree/main/packages/uipath-llamaindex/samples) | +| OpenAI Agents SDK | [Get Started](../openai-agents/quick_start.md) | [Samples](https://github.com/UiPath/uipath-integrations-python/tree/main/packages/uipath-openai-agents/samples) | +| PydanticAI | [README](https://github.com/UiPath/uipath-integrations-python/blob/main/packages/uipath-pydantic-ai/README.md) | [Samples](https://github.com/UiPath/uipath-integrations-python/tree/main/packages/uipath-pydantic-ai/samples) | +| Google ADK | [README](https://github.com/UiPath/uipath-integrations-python/blob/main/packages/uipath-google-adk/README.md) | [Samples](https://github.com/UiPath/uipath-integrations-python/tree/main/packages/uipath-google-adk/samples) | +| UiPath Agent Framework | [README](https://github.com/UiPath/uipath-integrations-python/blob/main/packages/uipath-agent-framework/README.md) | [Samples](https://github.com/UiPath/uipath-integrations-python/tree/main/packages/uipath-agent-framework/samples) | + + +--- + +## Pack & Publish + +The same CLI workflow applies as for coded functions: + + + +```shell +> uipath pack +⠋ Packaging project ... +Name : my-agent +Version : 0.1.0 +Description: Add your description here +Authors : Your Name +✓ Project successfully packaged. + +> uipath publish +⠋ Fetching available package feeds... +Select feed number: 0 +✓ Package published successfully! +``` + +After publishing, the agent registers as an Orchestrator Process and can be invoked from Maestro, the Orchestrator API, or the CLI. + +See [CLI Reference](../cli/index.md) for full `pack`, `publish`, and `invoke` options. + +--- + +## Studio Web Integration + +Connect your agent to a Studio Web solution for cloud debugging, evaluation, and solution packaging. + +See [Studio Web Integration](./studio_web.md) for setup and sync details. + +--- + +## Evaluations + +Coded agents support evaluations in Studio Web and locally via `uipath eval`. Evaluators cover LLM output quality, tool call correctness, and trajectory analysis. + +See the [Evaluations documentation](../eval/index.md) for available evaluators and how to define evaluation sets. diff --git a/packages/uipath/docs/core/assets/maestro_execution_trace_light.png b/packages/uipath/docs/core/assets/maestro_execution_trace_light.png new file mode 100644 index 000000000..d3d364168 Binary files /dev/null and b/packages/uipath/docs/core/assets/maestro_execution_trace_light.png differ diff --git a/packages/uipath/docs/core/assets/maestro_service_task_light.png b/packages/uipath/docs/core/assets/maestro_service_task_light.png new file mode 100644 index 000000000..314cb383e Binary files /dev/null and b/packages/uipath/docs/core/assets/maestro_service_task_light.png differ diff --git a/packages/uipath/docs/core/assets/orchestrator_processes_light.png b/packages/uipath/docs/core/assets/orchestrator_processes_light.png new file mode 100644 index 000000000..9b17f72cf Binary files /dev/null and b/packages/uipath/docs/core/assets/orchestrator_processes_light.png differ diff --git a/packages/uipath/docs/core/assets/studio_web_select_function_dark.png b/packages/uipath/docs/core/assets/studio_web_select_function_dark.png new file mode 100644 index 000000000..68a66fc92 Binary files /dev/null and b/packages/uipath/docs/core/assets/studio_web_select_function_dark.png differ diff --git a/packages/uipath/docs/core/assets/studio_web_select_function_light.png b/packages/uipath/docs/core/assets/studio_web_select_function_light.png new file mode 100644 index 000000000..617e30c2f Binary files /dev/null and b/packages/uipath/docs/core/assets/studio_web_select_function_light.png differ diff --git a/packages/uipath/docs/core/environment_variables.md b/packages/uipath/docs/core/environment_variables.md index 6c88d3532..bcf1bcc8b 100644 --- a/packages/uipath/docs/core/environment_variables.md +++ b/packages/uipath/docs/core/environment_variables.md @@ -17,12 +17,12 @@ UIPATH_FOLDER_PATH=/default/path export UIPATH_FOLDER_PATH=/system/path ``` /// warning -When deploying your agent to production, ensure that all required environment variables (such as API keys and custom configurations) are properly configured in your process settings. This step is crucial for the successful operation of your published package. +When deploying your project to production, ensure that all required environment variables (such as API keys and custom configurations) are properly configured in your process settings. This step is crucial for the successful operation of your published package. /// ## Design -Create a `.env` file in your project's root directory to manage environment variables locally. When using the `uipath auth` or `uipath new my-agent` commands, this file is automatically created. +Create a `.env` file in your project's root directory to manage environment variables locally. When using the `uipath auth` or `uipath new` commands, this file is automatically created. The `uipath auth` command automatically populates this file with essential variables: diff --git a/packages/uipath/docs/core/functions.md b/packages/uipath/docs/core/functions.md new file mode 100644 index 000000000..6c073f68f --- /dev/null +++ b/packages/uipath/docs/core/functions.md @@ -0,0 +1,423 @@ +# Python Coded Functions + +A coded function is Python code with typed input and output that runs as an Orchestrator job. You write plain Python — no agent framework, no LLM required — package it with the CLI, and invoke it from Maestro processes, Coded Apps, or any UiPath job trigger. + +Use coded functions for deterministic compute steps: document extraction, ERP writes, data validation, external API calls. Use a [coded agent](./agents.md) when your logic needs an LLM decision loop or a multi-step reasoning chain. + +!!! warning "Preview Feature" + This feature is in preview and is subject to changes. + +--- + +## Quickstart + +//// tab | uv + + + +```shell +> mkdir my-function && cd my-function +> uv init . --python 3.11 +> uv add uipath + +> uipath auth +⠋ Authenticating with UiPath ... +✓ Authentication successful. + +> uipath new my-function +✓ Created 'main.py' file. +✓ Created 'pyproject.toml' file. +✓ Created 'uipath.json' file. + +> uipath init +⠋ Initializing UiPath project ... +✓ Created 'entry-points.json' file. +✓ Created 'bindings.json' file. + +> uipath run main '{"message": "hello"}' +{"message": "hello"} +``` + +//// + +//// tab | pip + + + +```shell +> mkdir my-function && cd my-function +> python -m venv .venv +> source .venv/bin/activate +> pip install uipath + +> uipath auth +⠋ Authenticating with UiPath ... +✓ Authentication successful. + +> uipath new my-function +✓ Created 'main.py' file. +✓ Created 'pyproject.toml' file. +✓ Created 'uipath.json' file. + +> uipath init +⠋ Initializing UiPath project ... +✓ Created 'entry-points.json' file. +✓ Created 'bindings.json' file. + +> uipath run main '{"message": "hello"}' +{"message": "hello"} +``` + +//// + +--- + +## Project Structure + +``` +my-function/ +├── main.py # function logic +├── pyproject.toml # project metadata and dependencies +├── uipath.json # entry point declarations +├── entry-points.json # generated — I/O JSON Schema +└── bindings.json # generated — resource binding overrides +``` + +### `uipath.json` + +Declares which Python functions are callable entry points: + +```json +{ + "functions": { + "main": "main.py:main" + } +} +``` + +The key (`"main"`) is the entry point name used in CLI commands. The value (`"main.py:main"`) is `:`. + +### `pyproject.toml` + +```toml +[project] +name = "my-function" +version = "0.1.0" +description = "..." +authors = [{ name = "Your Name", email = "you@example.com" }] +requires-python = ">=3.11" +dependencies = ["uipath>=2.0"] +``` + +Standard project metadata and dependencies. The `functions` map in `uipath.json` (above) is what marks the project as a coded function — `pyproject.toml` needs no UiPath-specific entries. + +### Generated files + +| File | Purpose | +|------|---------| +| `entry-points.json` | Input/output JSON Schema derived from your `Input`/`Output` models — used by Maestro for variable binding | +| `bindings.json` | Resource binding overrides (assets, connections, buckets) for local development | + +/// warning +`uipath init` executes your entrypoint Python file(s) (as declared in `uipath.json`, e.g., `main.py`) to derive the I/O schema. Re-run it after every change to your `Input` or `Output` models. +/// + +--- + +## Input & Output + +Define `Input` and `Output` as typed Python — a stdlib `@dataclass`, a pydantic `BaseModel`, or `pydantic.dataclasses.dataclass`. The runtime uses these type hints to parse the invocation payload and exports them as JSON Schema for Maestro variable binding. The entry point can be a sync `def` or an `async def` — both are supported. + +```python +from dataclasses import dataclass, field +from typing import Any + + +@dataclass +class Input: + document_id: str = "" + amount: float = 0.0 + + +@dataclass +class Output: + result_id: str = "" + status: str = "" + error_type: str = "" + error_message: str = "" + + +def main(input: Input) -> Output: + ... +``` + +### Supported types + +| Python type | Notes | +|-------------|-------| +| `str`, `int`, `float`, `bool` | Primitives | +| `list[str]`, `list[dict]` | Arrays | +| `dict[str, Any]` | Freeform object | +| Nested `@dataclass` | Becomes a nested JSON object | +| `X \| None`, `Optional[X]` | Nullable field | + +### Error output pattern + +Return business errors as typed output fields rather than raising exceptions. This lets Maestro inspect the error reason and route the process accordingly: + +```python +@dataclass +class Output: + bill_id: str = "" + error_type: str = "" # e.g. "VENDOR_NOT_FOUND", "VALIDATION_ERROR" + error_message: str = "" # human-readable detail + + +def main(input: Input) -> Output: + try: + bill_id = create_vendor_bill(input) + return Output(bill_id=bill_id) + except VendorNotFoundError as exc: + return Output(error_type="VENDOR_NOT_FOUND", error_message=str(exc)) + except Exception as exc: + return Output(error_type="FAILED", error_message=str(exc)) +``` + +Reserve `raise` for unrecoverable infrastructure failures (network timeout, authentication error) that should mark the Orchestrator job as faulted. + +--- + +## Platform Services + +`UiPath()` gives your function access to Orchestrator resources at runtime. Credentials are injected automatically when running as a job — no configuration needed. + +```python +from uipath.platform import UiPath + +sdk = UiPath() +``` + +### Assets + +Read credential and configuration values stored in Orchestrator: + +```python +# String asset +asset = sdk.assets.retrieve("API_BASE_URL", folder_path="Shared") +base_url = str(asset.string_value or "") + +# Credential asset +creds = sdk.assets.retrieve("ERP_CREDENTIALS", folder_path="Shared") +username = str(creds.credential_username or "") +password = str(creds.credential_password or "") +``` + +See [Assets](./assets.md) for the full API reference. + +### Buckets + +Download and upload files: + +```python +# Download +sdk.buckets.download( + name="Invoices", + blob_file_path="incoming/acme-001.pdf", + destination_path="/tmp/acme-001.pdf", + folder_path="Shared", +) + +# Upload +sdk.buckets.upload( + name="Processed", + blob_file_path="results/acme-001-result.json", + content_file_path="/tmp/result.json", + folder_path="Shared", +) +``` + +See [Buckets](./buckets.md) for the full API reference. + +### Connections + +Access Integration Service connections for ERP and SaaS systems: + +```python +from uipath.platform.connections.connections import ActivityMetadata, ActivityParameterLocationInfo + +conn = sdk.connections.retrieve("your-connection-id") + +result = sdk.connections.invoke_activity( + activity_metadata=ActivityMetadata( + object_path="/your-endpoint", + method_name="POST", + content_type="application/json", + parameter_location_info=ActivityParameterLocationInfo(body_fields=["query"]), + ), + connection_id="your-connection-id", + activity_input={"query": "SELECT id FROM records LIMIT 10"}, +) +``` + +See [Connections](./connections.md) for the full API reference. + +--- + +## Tracing + +Use `@traced` to make individual steps visible as spans in the Orchestrator job trace view and Maestro dashboards. + +```python +from uipath.tracing import traced + + +@traced(name="fetch_document", run_type="uipath") +def fetch_document(document_id: str) -> bytes: + ... + + +@traced(name="extract_fields", run_type="uipath") +def extract_fields(content: bytes) -> dict: + ... + + +@traced(name="post_to_erp", run_type="uipath") +def post_to_erp(data: dict) -> str: + ... + + +def main(input: Input) -> Output: # entry point — NOT traced + content = fetch_document(input.document_id) + data = extract_fields(content) + result_id = post_to_erp(data) + return Output(result_id=result_id) +``` + +/// warning +Do not apply `@traced` to the entry point function. The Orchestrator runtime wraps the entire job in its own span — adding a second trace on the entry point creates a duplicate outer span. +/// + +Use `hide_input=True` or `hide_output=True` to redact sensitive data from trace storage: + +```python +@traced(name="get_api_token", run_type="uipath", hide_input=True, hide_output=True) +def get_api_token(client_id: str, client_secret: str) -> str: + ... +``` + + + Maestro execution trail showing @traced sub-step spans (assets_retrieve, ixp_digitize, netsuite_get_vendor, etc.) with durations and parent-child nesting under a Service Task + + +See [Tracing](./traced.md) for the full decorator reference. + +--- + +## Multiple Entry Points + +One project can expose several callable functions, each with its own `Input`/`Output`. Define them in `uipath.json`: + +```json +{ + "functions": { + "extract": "main.py:extract_data", + "validate": "main.py:validate_data", + "post_erp": "main.py:post_to_erp" + } +} +``` + +Run `uipath init` after adding new entry points. Each can be invoked independently: + + + +```shell +> uipath run extract '{"document_id": "invoice-001.pdf"}' +> uipath run validate '{"vendor_name": "Acme", "total": 1234.56}' +> uipath run post_erp '{"bill_id": "12345"}' +``` + +Each entry point publishes as a separate invocable function in Orchestrator. + +--- + +## Idempotency + +Functions may be retried by Maestro after a transient failure. Always check for an existing result before writing to an external system: + +```python +@traced(name="find_existing", run_type="uipath") +def find_existing(invoice_number: str) -> str | None: + # query external system by stable business key + ... + + +def main(input: Input) -> Output: + existing_id = find_existing(input.invoice_number) + if existing_id: + return Output(result_id=existing_id, status="Already Processed") + + result_id = create_record(input) + return Output(result_id=result_id, status="Created") +``` + +Use a stable, business-meaningful identifier (invoice number, order ID) as the idempotency key — avoid auto-generated IDs that don't exist before the first write. + +--- + +## Pack & Publish + + + +```shell +> uipath pack +⠋ Packaging project ... +Name : my-function +Version : 0.1.0 +Description: ... +Authors : Your Name +✓ Project successfully packaged. + +> uipath publish +⠋ Fetching available package feeds... +👇 Select package feed: + 0: Orchestrator Tenant Processes Feed + 1: Orchestrator Personal Workspace Feed +Select feed number: 0 +✓ Package published successfully! +``` + +After publishing, the function registers as an **Orchestrator Process**. It can then be: + +- Invoked as a **Maestro Service Task** — Maestro binds typed input/output to process variables automatically from the exported JSON Schema +- Triggered via the **Orchestrator API** (`POST /Jobs/StartJobs`) +- Run from the CLI: `uipath invoke main '{"..."}'` +- Started from a **Studio workflow** using the **Run Job** activity + + + Coded functions published as Function (python) type in the Orchestrator Processes list + + + Coded function wired as a Maestro Service Task with typed input/output variable binding + + +See [CLI Reference](../cli/index.md) for full `pack`, `publish`, and `invoke` options. + +--- + +## Studio Web Integration + +Connect your function to a Studio Web solution for cloud debugging or solution packaging: + + + +```shell +> uipath push +Pushing UiPath project to Studio Web... +Uploading 'main.py' +Uploading 'uipath.json' +Updating 'pyproject.toml' +✓ Project pushed successfully +``` + +See [Studio Web Integration](./studio_web.md) for setup and sync details. diff --git a/packages/uipath/docs/core/getting_started.md b/packages/uipath/docs/core/getting_started.md index 6c00b0089..50da82107 100644 --- a/packages/uipath/docs/core/getting_started.md +++ b/packages/uipath/docs/core/getting_started.md @@ -114,6 +114,10 @@ Upon successful authentication, your project will contain a `.env` file with you ### Writing Your Code +/// tip +This walkthrough creates a **coded function** — plain Python with typed input and output, no LLM required. For a complete reference including platform services, tracing, idempotency, and Maestro integration, see [Python Coded Functions](./functions.md). +/// + Open `main.py` in your code editor. You can start with this example code: ```python from dataclasses import dataclass diff --git a/packages/uipath/docs/core/guardrails.md b/packages/uipath/docs/core/guardrails.md index 40aa69abc..6a220bfe2 100644 --- a/packages/uipath/docs/core/guardrails.md +++ b/packages/uipath/docs/core/guardrails.md @@ -1 +1,484 @@ +# Guardrails + +Guardrails are safeguards applied before and/or after execution to inspect inputs and outputs for policy violations — PII, harmful content, prompt attacks, intellectual property, and custom rules — and respond by logging, blocking, or modifying the data. + +They can be applied at three scopes: + +- **Tool** — individual tool functions called by an agent +- **LLM** — LLM factory functions or chat model objects (e.g. LangChain `BaseChatModel`) +- **Agent** — agent-level methods and nodes + +The `@guardrail` decorator works with plain Python functions, async functions, and any LangChain/LangGraph object recognised by a registered framework adapter. + +## Usage + +Apply the `@guardrail` decorator to any callable — tool functions, LLM factories, agent factories, or async agent nodes. The decorator intercepts calls at the configured stage and evaluates the data against the provided validator. + +**Tool function:** + +```python +from uipath.platform.guardrails import ( + BlockAction, + GuardrailExecutionStage, + PIIDetectionEntity, + PIIDetectionEntityType, + PIIValidator, + guardrail, +) + +@guardrail( + validator=PIIValidator( + entities=[PIIDetectionEntity(PIIDetectionEntityType.EMAIL, threshold=0.5)] + ), + action=BlockAction(), + name="No PII in output", + stage=GuardrailExecutionStage.POST, +) +def analyze_joke(joke: str) -> str: + ... +``` + +When using LangChain's `@tool`, `@guardrail` must be placed **above** `@tool`: + +```python +from langchain_core.tools import tool + +@guardrail( + validator=PIIValidator( + entities=[PIIDetectionEntity(PIIDetectionEntityType.EMAIL, threshold=0.5)] + ), + action=BlockAction(), + name="No PII in tool input", + stage=GuardrailExecutionStage.PRE, +) +@tool # @guardrail wraps the already-decorated tool object +def analyze_joke(joke: str) -> str: + ... +``` + +**LLM factory function:** + +```python +@guardrail( + validator=UserPromptAttacksValidator(), + action=BlockAction(), + name="LLM User Prompt Attacks Detection", + stage=GuardrailExecutionStage.PRE, +) +def create_llm(): + return UiPathChat(model="gpt-4o-2024-08-06") +``` + +**Agent factory or async node:** + +```python +@guardrail( + validator=PIIValidator( + entities=[PIIDetectionEntity(PIIDetectionEntityType.PERSON, threshold=0.5)] + ), + action=BlockAction( + title="Person name detection", + detail="Person name detected and is not allowed", + ), + name="Agent PII Detection", + stage=GuardrailExecutionStage.PRE, +) +async def joke_node(state: Input) -> Output: + ... +``` + +## Execution Stages + +The `stage` parameter controls when the guardrail evaluates. Not all validators support all stages. + +| Stage | When evaluated | Supported by | +|-------|---------------|--------------| +| `PRE` | Before the function runs | All validators | +| `POST` | After the function runs | All except `UserPromptAttacksValidator` | +| `PRE_AND_POST` | Both before and after | `PIIValidator`, `HarmfulContentValidator`, `LLMAsJudgeValidator`, `CustomValidator` | + +## Built-in Validators + +Built-in validators are backed by the UiPath Guardrails API (powered by Azure Content Safety). They require a UiPath connection with the appropriate entitlements. + +### PII Detection + +Detects personally identifiable information in text. Supports 18 entity types with per-entity confidence thresholds. + +```python +from uipath.platform.guardrails import ( + BlockAction, + PIIDetectionEntity, + PIIDetectionEntityType, + PIIValidator, + guardrail, +) + +@guardrail( + validator=PIIValidator( + entities=[ + PIIDetectionEntity(name=PIIDetectionEntityType.EMAIL, threshold=0.7), + PIIDetectionEntity(name=PIIDetectionEntityType.PHONE_NUMBER, threshold=0.5), + PIIDetectionEntity(name=PIIDetectionEntityType.US_SOCIAL_SECURITY_NUMBER), + ] + ), + action=BlockAction(), + name="No PII", +) +def process_document(content: str) -> str: + ... +``` + +`threshold` is a confidence value between `0.0` and `1.0` (default `0.5`). Lower values increase sensitivity. + +### Harmful Content + +Detects harmful or unsafe content across four Azure Content Safety categories. Each category has a severity threshold from `0` (most sensitive) to `6` (least sensitive), defaulting to `2`. + +```python +from uipath.platform.guardrails import ( + BlockAction, + HarmfulContentEntity, + HarmfulContentEntityType, + HarmfulContentValidator, + guardrail, +) + +@guardrail( + validator=HarmfulContentValidator( + entities=[ + HarmfulContentEntity(name=HarmfulContentEntityType.VIOLENCE, threshold=2), + HarmfulContentEntity(name=HarmfulContentEntityType.HATE, threshold=2), + ] + ), + action=BlockAction(), + name="Safe content only", +) +def generate_response(prompt: str) -> str: + ... +``` + +### User Prompt Attacks + +Detects adversarial user prompt patterns (e.g. jailbreak attempts). No configuration parameters required. Restricted to `PRE` stage only. + +```python +from uipath.platform.guardrails import ( + BlockAction, + GuardrailExecutionStage, + UserPromptAttacksValidator, + guardrail, +) + +@guardrail( + validator=UserPromptAttacksValidator(), + action=BlockAction(), + name="No prompt attacks", + stage=GuardrailExecutionStage.PRE, +) +def chat(message: str) -> str: + ... +``` + +### Intellectual Property + +Detects potential intellectual property violations in generated output. Restricted to `POST` stage only — this is an output concern. + +```python +from uipath.platform.guardrails import ( + BlockAction, + GuardrailExecutionStage, + IntellectualPropertyEntityType, + IntellectualPropertyValidator, + guardrail, +) + +@guardrail( + validator=IntellectualPropertyValidator( + entities=[ + IntellectualPropertyEntityType.TEXT, + IntellectualPropertyEntityType.CODE, + ] + ), + action=BlockAction(), + name="No IP violations", + stage=GuardrailExecutionStage.POST, +) +def generate_code(spec: str) -> str: + ... +``` + +### LLM-as-judge + +Evaluates content against a rule written in plain language, using a judge LLM to decide whether the payload complies. Use it for policy checks that are hard to express as fixed entities or rules — tone, topicality, disclaimers, domain-specific policies. + +```python +from uipath.platform.guardrails import ( + BlockAction, + GuardrailExecutionStage, + LLMAsJudgeValidator, + guardrail, +) + +@guardrail( + validator=LLMAsJudgeValidator( + guardrail_text=( + "The response must remain professional and must not contain " + "financial or investment advice." + ), + model="gpt-4o-2024-08-06", + threshold=2, + ), + action=BlockAction(), + name="No financial advice", + stage=GuardrailExecutionStage.POST, +) +def answer_question(question: str) -> str: + ... +``` + +- `guardrail_text` (required) — the rule the judge evaluates against, at most 4000 characters. +- `model` (required) — the judge model id, e.g. `"gpt-4o-2024-08-06"`. It must be a model your organization's governance policy allows for the LLM-as-judge guardrail — LLM Gateway enforces the permitted list, so a model that isn't authorized for judging is rejected. +- `threshold` — strictness from `0` (strictest) to `6` (most lenient), defaulting to `2`. Higher values flag only clear violations. +- `positive_examples` / `negative_examples` — optional example payloads (not descriptions) that comply with / violate the rule, used to calibrate the judge. At most 2 entries per list, each at most 1000 characters. + +## Actions + +Actions define what happens when a violation is detected. + +### LogAction + +Logs the violation and lets execution continue. The original data is unchanged. + +```python +from uipath.platform.guardrails import LogAction, LoggingSeverityLevel + +action = LogAction(severity_level=LoggingSeverityLevel.WARNING) +action = LogAction(severity_level=LoggingSeverityLevel.ERROR, message="PII found in output") +``` + +### BlockAction + +Raises `GuardrailBlockException` to stop execution immediately. Framework adapters (e.g. LangChain) catch this exception and convert it to their own error type. + +```python +from uipath.platform.guardrails import BlockAction + +action = BlockAction() +action = BlockAction(title="PII detected", detail="Email address found in response") +``` + +### Custom Actions + +Subclass `GuardrailAction` to implement custom behaviour, such as content sanitisation: + +```python +from typing import Any +from uipath.core.guardrails import GuardrailValidationResult +from uipath.platform.guardrails import GuardrailAction + +class RedactAction(GuardrailAction): + def handle_validation_result( + self, + result: GuardrailValidationResult, + data: str | dict[str, Any], + guardrail_name: str, + ) -> str | dict[str, Any] | None: + # Return modified data to replace the original, or None to leave unchanged + if isinstance(data, str): + return "[REDACTED]" + return None +``` + +## Custom Validators + +`CustomValidator` applies an in-process rule function without any API call. The rule receives the input dict (PRE stage) or both input and output dicts (POST stage), and returns `True` to signal a violation. + +```python +from uipath.platform.guardrails import BlockAction, CustomValidator, guardrail + +@guardrail( + validator=CustomValidator(rule=lambda data: "forbidden" in str(data).lower()), + action=BlockAction(), + name="No forbidden words", +) +def my_tool(text: str) -> str: + ... +``` + +For POST-stage rules, accept two parameters to inspect both input and output: + +```python +def check_output(input_data: dict, output_data: dict) -> bool: + # Return True to trigger the guardrail + return len(output_data.get("response", "")) > 5000 + +@guardrail( + validator=CustomValidator(rule=check_output), + action=BlockAction(detail="Response exceeds maximum length"), + name="Length limit", + stage=GuardrailExecutionStage.POST, +) +def summarize(query: str) -> dict: + ... +``` + +For full control, subclass `CustomGuardrailValidator` directly. + +## Excluding Parameters + +Use `GuardrailExclude` with `Annotated` to prevent a specific parameter from being included in the guardrail evaluation payload. Useful for internal context objects, credentials, or other data that should never be inspected. + +```python +from typing import Annotated +from uipath.platform.guardrails import BlockAction, GuardrailExclude, PIIValidator, guardrail + +@guardrail( + validator=PIIValidator(entities=[PIIDetectionEntity(name=PIIDetectionEntityType.EMAIL)]), + action=BlockAction(), + name="No PII", +) +def process( + user_message: str, + internal_config: Annotated[dict, GuardrailExclude()], # excluded from guardrail +) -> str: + ... +``` + +## Stacking Guardrails + +Multiple `@guardrail` decorators can be stacked on the same function. Each is evaluated independently at its configured stage. + +```python +@guardrail( + validator=UserPromptAttacksValidator(), + action=BlockAction(), + name="No prompt attacks", + stage=GuardrailExecutionStage.PRE, +) +@guardrail( + validator=PIIValidator(entities=[PIIDetectionEntity(name=PIIDetectionEntityType.EMAIL)]), + action=LogAction(), + name="PII audit", + stage=GuardrailExecutionStage.POST, +) +def handle_request(user_input: str) -> str: + ... +``` + +## Low-level API + +For direct programmatic use without the decorator, the `GuardrailsService` is available on the `UiPath` client: + +```python +from uipath.platform import UiPath +from uipath.platform.guardrails import BuiltInValidatorGuardrail + +sdk = UiPath() +result = sdk.guardrails.evaluate_guardrail( + input_data="Contact me at user@example.com", + guardrail=BuiltInValidatorGuardrail( + id="my-guardrail", + name="PII check", + guardrail_type="builtInValidator", + validator_type="pii_detection", + ), +) +print(result.result, result.reason) +``` + +--- + +## API Reference + +### Service + ::: uipath.platform.guardrails._guardrails_service + options: + members: + - GuardrailsService + +### Decorator + +::: uipath.platform.guardrails.decorators._guardrail + options: + members: + - guardrail + +### Execution Stage + +::: uipath.platform.guardrails.decorators._enums + options: + members: + - GuardrailExecutionStage + - PIIDetectionEntityType + - HarmfulContentEntityType + - IntellectualPropertyEntityType + +### Actions + +::: uipath.platform.guardrails.decorators._models + options: + members: + - GuardrailAction + - PIIDetectionEntity + - HarmfulContentEntity + +::: uipath.platform.guardrails.decorators._actions + options: + members: + - LoggingSeverityLevel + - LogAction + - BlockAction + +::: uipath.platform.guardrails.decorators._exceptions + options: + members: + - GuardrailBlockException + +### Exclude Marker + +::: uipath.platform.guardrails.decorators._core + options: + members: + - GuardrailExclude + +### Validators + +::: uipath.platform.guardrails.decorators.validators._base + options: + members: + - GuardrailValidatorBase + - BuiltInGuardrailValidator + - CustomGuardrailValidator + +::: uipath.platform.guardrails.decorators.validators.pii + options: + members: + - PIIValidator + +::: uipath.platform.guardrails.decorators.validators.harmful_content + options: + members: + - HarmfulContentValidator + +::: uipath.platform.guardrails.decorators.validators.intellectual_property + options: + members: + - IntellectualPropertyValidator + +::: uipath.platform.guardrails.decorators.validators.llm_as_judge + options: + members: + - LLMAsJudgeValidator + +::: uipath.platform.guardrails.decorators.validators.user_prompt_attacks + options: + members: + - UserPromptAttacksValidator + +::: uipath.platform.guardrails.decorators.validators.custom + options: + members: + - CustomValidator + - RuleFunction diff --git a/packages/uipath/docs/core/release_notes.md b/packages/uipath/docs/core/release_notes.md index 325baad92..5565f3689 100644 --- a/packages/uipath/docs/core/release_notes.md +++ b/packages/uipath/docs/core/release_notes.md @@ -1,126 +1,59 @@ -# 🚨 Breaking Changes for UiPath Python SDK (v2.2.0+) - -**Release Date:** November 26, 2025 - -Version 2.2.0 of the **UiPath Python SDK** introduces several breaking changes affecting both the SDK and CLI. - -## Breaking Changes - -### 1. Minimum Python Version: 3.11+ Required - -**What's changing:** Python 3.10 is no longer supported for `uipath-python`, `uipath-langchain-python`, `uipath-llamaindex-python`. - -**Action required:** Upgrade to Python 3.11 or higher. - -### 2. Import Path Change - -**What's changing:** The `UiPath` class has moved from `uipath` to `uipath.platform`. - -**Action required:** Update your imports: - -```python -# Before -from uipath import UiPath -from uipath.models import Job, Asset, Queue -from uipath.models import Entity - -# After -from uipath.platform import UiPath, Job, Asset, Queue - -client = UiPath(...) -``` - -### 3. Transition to LangChain v1 (for `uipath-langchain` only) - -**What's changing:** Minimum required versions are now LangChain 1.0.0+ and LangGraph 1.0.0+ - -**Action required:** Review and update your code according to the [LangChain v1 Migration Guide](https://docs.langchain.com/oss/python/migrate/langchain-v1). - -**Note:** This only applies if you're using the `uipath-langchain` package. - -### 4. Configuration Architecture Redesign - -We've restructured how UiPath projects define and manage their resources: - -**`uipath.json` - Configuration File (Updated Purpose)** -- Previously contained entrypoints and bindings; now serves as a streamlined configuration file -- For **pure Python scripts**, define entrypoints in the `functions` section: - ```json - { - "functions": { - "entrypoint1": "src/main.py:", - "entrypoint2": "src/graph.py:runtime" - } - } - ``` -- For **LangGraph graphs**, define entrypoints in `langgraph.json` (same as before) -- For **LlamaIndex workflows**, define entrypoints in `llamaindex.json` (same as before) - -**`bindings.json` - Manual Binding Definitions (New)** -- Overridable resources (bindings) now stored in a separate file -- Bindings are **no longer automatically inferred** from code -- Must be manually defined by the user for now (we're working on an interactive configurator to simplify this process) +--- +title: Release Notes +--- -**`entry-points.json` - I/O Schema (New)** -- Contains the input/output schema for your entrypoints -- Automatically inferred from code based on entrypoints defined in `llamaindex.json`/`langgraph.json`/`uipath.json` +# Release Notes -## Migration Guide +A catalog of the releases most relevant to UiPath Python SDK users (breaking changes and notable updates). Full details live in each GitHub release, linked below. -### Stay on v2.1.x +## `uipath` (SDK & CLI) -To avoid these breaking changes and keep your current setup, pin your dependency in `pyproject.toml`: +| Release | Date | What's relevant | Notes | +|---------|------|-----------------|-------| +| [v2.12.0](https://github.com/UiPath/uipath-python/releases/tag/v2.12.0) | 2026-06-30 | Eval framework: `AgentExecution` → `WorkloadExecution`; fields `agent_trace`/`agent_output` → `workload_trace`/`workload_output`; `evaluate()`/`validate_and_evaluate_criteria()` keyword `agent_execution` → `workload_execution` | 🚨 Breaking — see [migration](#migration-v2120-eval-workloadexecution-rename) | +| [v2.10.0](https://github.com/UiPath/uipath-python/releases/tag/v2.10.0) | 2026-02-27 | Coded function schema `type` changed from `"agent"` to `"function"` | 🚨 Breaking | +| [v2.9.0](https://github.com/UiPath/uipath-python/releases/tag/v2.9.0) | 2026-02-23 | `platform` extracted to `uipath-platform`, context grounding contract changes, `uipath dev` defaults to `web` | 🚨 Breaking | +| [v2.2.0](https://github.com/UiPath/uipath-python/releases/tag/v2.2.0) | 2025-11-26 | Python 3.11+ required, `UiPath` import moved to `uipath.platform`, configuration architecture redesign | 🚨 Breaking | -```toml -"uipath>=2.1.x,<2.2.0" -``` +## `uipath-langchain` -**For `uipath-langchain` users:** To stay on the current version without LangChain v1: -```toml -"uipath-langchain>=0.0.x,<0.1.0" -``` +| Release | Date | What's relevant | Notes | +|---------|------|-----------------|-------| +| [v0.10.0](https://github.com/UiPath/uipath-langchain-python/releases/tag/v0.10.0) | 2026-04-23 | Transport/auth split into new `uipath-llm-client` and `uipath-langchain-client` packages (legacy preserved) | Non-breaking | -### Migrate to v2.2.0+ +## `uipath-runtime` -1. **Upgrade to v2.2.0+** - - Update the dependencies in `pyproject.toml` with: - ```toml - "uipath>=2.2.x,<2.3.0" - ``` +| Release | Date | What's relevant | Notes | +|---------|------|-----------------|-------| +| [v0.3.0](https://github.com/UiPath/uipath-runtime-python/releases/tag/v0.3.0) | 2025-12-18 | `UiPathDebugBridgeProtocol` renamed to `UiPathDebugProtocol` | 🚨 Breaking (protocol implementers only) | - Bounding the version to <2.3.0 prevents future breaking changes - - **For `uipath-langchain` users:** - To migrate to LangChain v1: - ```toml - "uipath-langchain>=0.1.0,<0.2.0" - ``` - **For `uipath-langchain`/`uipath-llamaindex` users:** - Make sure to also reference `uipath` in your `pyproject.toml` - future versions will no longer reference the main `uipath` CLI package as a dependency. +## Migration: v2.12.0 (eval `WorkloadExecution` rename) -2. **Upgrade the Python version to 3.11+** - - In `pyproject.toml` specify the required Python version by adding or updating the following field: - ```toml - requires-python = ">=3.11" - ``` +The eval framework's central execution type and its keyword parameters were renamed to +unified-evals terminology. The value passed to every evaluator represents *a workload +execution* (agent, process/orchestration, case management, …), not specifically an agent. -3. **Update imports** - - Change `from uipath import UiPath` to `from uipath.platform import UiPath`. +These are breaking changes for code that uses the affected public names. Update as follows: -4. **Review LangChain v1 changes (if using `uipath-langchain`)** - - Review the [LangChain v1 Migration Guide](https://docs.langchain.com/oss/python/migrate/langchain-v1) and update your code accordingly. +| Before | After | +|--------|-------| +| `from uipath.eval.models import AgentExecution` | `from uipath.eval.models import WorkloadExecution` | +| `WorkloadExecution(agent_trace=...)` | `WorkloadExecution(workload_trace=...)` | +| `WorkloadExecution(agent_output=...)` | `WorkloadExecution(workload_output=...)` | +| `execution.agent_trace` / `execution.agent_output` | `execution.workload_trace` / `execution.workload_output` | +| `evaluator.evaluate(agent_execution=...)` | `evaluator.evaluate(workload_execution=...)` | +| `evaluator.validate_and_evaluate_criteria(agent_execution=...)` | `evaluator.validate_and_evaluate_criteria(workload_execution=...)` | -5. **Update configuration files** - - - **Define your entrypoints** in `scripts` within `uipath.json` (not applicable if you already use `langgraph.json`/`llamaindex.json`) - - **Run `uipath init`** to automatically generate the `entry-points.json` I/O schema from your configuration - - **Create `bindings.json`** and manually define all overridable resources - - **Important:** If you update your script/agent code, run `uipath init` again to regenerate the I/O schema +What still works: ---- +- **`AgentExecution` (the class name)** is soft-deprecated: importing it still resolves to + `WorkloadExecution` and emits a `DeprecationWarning` (removed in **uipath 3.0**). Note the + *fields* are not aliased — constructing `AgentExecution(agent_output=..., agent_trace=...)` + now raises a `ValidationError`; rename the fields as above. +- **Custom evaluators that override `evaluate` / `validate_and_evaluate_criteria` with the old + `agent_execution` parameter name** keep running, because the runtime dispatches positionally. + Only *callers* passing `agent_execution=` by keyword are affected. -For questions or issues, please open a ticket: [UiPath Python SDK Submit Issue](https://github.com/UiPath/uipath-python/issues) \ No newline at end of file +Unchanged (intentionally): the `WorkloadExecution.agent_input` and +`expected_agent_behavior` fields, and the `agent_output` / `agent_execution_time` fields on +`EvalRunUpdatedEvent` and `StudioWebProgressItem` (the latter a Studio Web wire contract). diff --git a/packages/uipath/docs/core/studio_web.md b/packages/uipath/docs/core/studio_web.md index 762735e48..09286dd73 100644 --- a/packages/uipath/docs/core/studio_web.md +++ b/packages/uipath/docs/core/studio_web.md @@ -1,12 +1,15 @@ # Studio Web Integration -[Studio Web](https://docs.uipath.com/studio-web/automation-cloud/latest/user-guide/overview) is a cloud IDE for building projects such as RPAs, low code agents, and API workflows. It also supports importing coded agents built locally. Bringing your coded agent into Studio Web gives you: +[Studio Web](https://docs.uipath.com/studio-web/automation-cloud/latest/user-guide/overview) is a cloud IDE for building projects such as RPAs, low code agents, and API workflows. It also supports importing coded agents and coded functions built locally. Bringing your project into Studio Web gives you: - Cloud debugging with dynamic breakpoints -- Running and defining evaluations directly in the cloud +- Running and defining evaluations directly in the cloud (coded agents only) - A unified build experience alongside multiple project types - Self contained solution deployment units +!!! warning "Preview Feature" + Coded function support is in preview and is subject to changes. + Coded agent in Studio Web -There are two ways to connect a coded agent to Studio Web: using a [Cloud Workspace](#cloud-workspace) or a [Local Workspace](#local-workspace). +There are two ways to connect your project to Studio Web: using a [Cloud Workspace](#cloud-workspace) or a [Local Workspace](#local-workspace). --- @@ -28,10 +31,14 @@ There are two ways to connect a coded agent to Studio Web: using a [Cloud Worksp In a Cloud Workspace, your project lives in Studio Web and you sync code between your local IDE and the cloud. -### Importing a Coded Agent +### Importing a Coded Agent or Coded Function 1. Open your solution in Studio Web -2. Create a new Agent and select **Coded** +2. Create the project: + + //// tab | Agent + + Create a new Agent and select **Coded**: -3. Choose a sample project to start from, or push an existing local agent - -### Pushing an Existing Agent + //// -If you already have a coded agent locally, you can sync it to Studio Web: + //// tab | Function -1. Copy the `UIPATH_PROJECT_ID` from Studio Web into your `.env` file + Use the **Initial setup screen** to get started: - + + //// + +3. Choose a sample project to start from, or push an existing local project + +### Pushing an Existing Project + +If you already have a project locally, you can sync it to Studio Web: + +1. Copy the `UIPATH_PROJECT_ID` from Studio Web into your `.env` file + + + + + + 2. Push your project: + ```shell > uipath push Pushing UiPath project to Studio Web... @@ -79,7 +107,7 @@ If you already have a coded agent locally, you can sync it to Studio Web: 🔵 Resource import summary: 3 total resources - 1 created, 1 updated, 1 unchanged, 0 not found ``` - Notice the **Resource import summary** at the end. The push command also imports resources defined in `bindings.json` into the Studio Web solution, just like importing resources for a low code agent. This ensures that all required resources are packaged with the solution, so the coded agent works anywhere the solution is deployed. + Notice the **Resource import summary** at the end. The push command also imports resources defined in `bindings.json` into the Studio Web solution, just like importing resources for a low code agent. This ensures that all required resources are packaged with the solution, so the project works anywhere the solution is deployed. See [`uipath push`](../cli/index.md) in the CLI Reference. @@ -88,6 +116,7 @@ If you already have a coded agent locally, you can sync it to Studio Web: To pull the latest version from Studio Web to your local environment: + ```shell > uipath pull Pulling UiPath project from Studio Web... @@ -116,21 +145,24 @@ See [`uipath pull`](../cli/index.md) in the CLI Reference. In a Local Workspace, your project lives on your machine and is linked to a Studio Web solution. See the [Local Workspace documentation](https://docs.uipath.com/studio-web/automation-cloud/latest/user-guide/solutions-in-the-local-workspace) for setup details. -You can either start from a predefined template in Studio Web or set up a new agent from scratch. +You can either start from a predefined template in Studio Web or set up a new project from scratch. ### Starting from a Template -When creating a new Coded agent in Studio Web with a Local Workspace, you can pick one of the predefined templates. This creates the project files directly on your machine. Templates come with sample code and predefined evaluations you can run immediately. +When creating a new coded agent or coded function in Studio Web with a Local Workspace, you can pick one of the predefined templates. This creates the project files directly on your machine. Templates come with sample code and predefined evaluations you can run immediately. -### Setting Up a New Agent +### Setting Up a New Project -You can also create a coded agent from scratch in your local IDE and have it appear in Studio Web. +You can also create a project from scratch in your local IDE and have it appear in Studio Web. + +#### Coded Agent First, install the SDK package for the framework you want to use: //// tab | uv + ```shell # Pick the package that matches your framework: # uipath-langchain - LangChain / LangGraph @@ -149,6 +181,7 @@ Installed 42 packages in 0.8s //// tab | pip + ```shell # Pick the package that matches your framework: # uipath-langchain - LangChain / LangGraph @@ -166,6 +199,7 @@ Successfully installed uipath-langchain Then authenticate, scaffold the agent, and initialize the project: + ```shell > uipath auth ⠋ Authenticating with UiPath ... @@ -187,57 +221,133 @@ Selected tenant: Tenant1 That's it, your agent should now be visible in Studio Web. +#### Coded Function + +A coded function doesn't require an additional framework package. Authenticate, scaffold the project, and initialize it: + + + +```shell +> uipath auth +⠋ Authenticating with UiPath ... +🔗 If a browser window did not open, please open the following URL in your browser: [LINK] +👇 Select tenant: + 0: Tenant1 + 1: Tenant2 +Select tenant number: 0 +Selected tenant: Tenant1 +✓ Authentication successful. + +> uipath new my-function +✓ Created 'main.py' file. +✓ Created 'pyproject.toml' file. +✓ Created 'uipath.json' file. + +> uipath init +⠋ Initializing UiPath project ... +✓ Created 'entry-points.json' file. +``` + +That's it, your coded function should now be visible in Studio Web. + --- ## Publishing -Once your coded agent is in Studio Web, publishing works the same as any other project. Click **Publish** in Studio Web and it will be packaged and deployed through the standard workflow. +Once your project is in Studio Web, publishing works the same as any other project. Click **Publish** in Studio Web and it will be packaged and deployed through the standard workflow. --- ## Running and Debugging -Your agent can be run both in the cloud (via Studio Web) and locally using the CLI. +Your project can be run both in the cloud (via Studio Web) and locally using the CLI. + +The CLI commands below take the entrypoint name as the first argument. For a coded agent, this is the graph name declared in your framework's config (for example, `agent` in `langgraph.json`). For a coded function, this is the key declared in the `functions` map of `uipath.json` (for example, `main`). ### Running Locally +//// tab | Agent + + ```shell > uipath run agent '{"message": "hello"}' ``` +//// + +//// tab | Function + + + +```shell +> uipath run main '{"message": "hello"}' +``` + +//// + See [`uipath run`](../cli/index.md) in the CLI Reference. ### Debugging Locally Use `uipath debug` for an enhanced local debugging experience. Unlike `uipath run`, the debug command: -- Auto polls for trigger responses when the agent suspends (e.g., LangGraph interrupts) +- Auto polls for trigger responses when the project suspends (e.g., LangGraph interrupts) - Fetches binding overwrites from Studio Web (configurable in **Debug > Debug Configuration > Solution resources**) +//// tab | Agent + + ```shell > uipath debug agent '{"message": "hello"}' ``` +//// + +//// tab | Function + + + +```shell +> uipath debug main '{"message": "hello"}' +``` + +//// + See [`uipath debug`](../cli/index.md) in the CLI Reference. ### Evaluating Locally -Run evaluations against your agent using the CLI: +Run evaluations against your project using the CLI: + +//// tab | Agent + ```shell > uipath eval agent .\evaluations\eval-sets\faithfulness-multi-model.json ``` +//// + +//// tab | Function + + + +```shell +> uipath eval main .\evaluations\eval-sets\default.json +``` + +//// + See [`uipath eval`](../cli/index.md) in the CLI Reference and the [Evaluations documentation](../eval/index.md). --- ## Syncing Evaluations -Evaluations can be defined either in Studio Web or locally. They sync automatically when you use `uipath pull` and `uipath push`. +Evaluations can be defined either in Studio Web or locally, and sync automatically when you use `uipath pull` and `uipath push`. Defining and running evaluations in Studio Web is supported for coded agents only; coded functions can still be evaluated locally with `uipath eval`. /// note Custom evaluators must be created locally. See [Custom Evaluators](../eval/custom_evaluators.md) for details. diff --git a/packages/uipath/docs/core/traced.md b/packages/uipath/docs/core/traced.md index da8dbc5dc..195e5751a 100644 --- a/packages/uipath/docs/core/traced.md +++ b/packages/uipath/docs/core/traced.md @@ -71,9 +71,47 @@ def sensitive_operation(secret): - Regular functions (sync/async) - Generator functions (sync/async) -## Example with plain python agents +## Example with coded functions -When used with plain python agents please call `wait_for_tracers()` at the end of the script to ensure all traces are sent, if this is not called the agent could end without sending all the traces. +Apply `@traced` to individual steps inside your function. Do **not** trace the entry point — the Orchestrator runtime wraps the job execution in its own span, so decorating the entry point creates a duplicate outer span. + +```python hl_lines="4 9 14" +from uipath.tracing import traced + + +@traced(name="fetch_document", run_type="uipath") +def fetch_document(document_id: str) -> bytes: + ... + + +@traced(name="extract_fields", run_type="uipath") +def extract_fields(content: bytes) -> dict: + ... + + +@traced(name="post_to_erp", run_type="uipath") +def post_to_erp(data: dict) -> str: + ... + + +def main(input: Input) -> Output: # entry point — NOT traced + content = fetch_document(input.document_id) + data = extract_fields(content) + result_id = post_to_erp(data) + return Output(result_id=result_id) +``` + +Use `hide_input=True` or `hide_output=True` on steps that handle credentials or PII: + +```python hl_lines="1" +@traced(name="get_api_token", run_type="uipath", hide_input=True, hide_output=True) +def get_api_token(client_id: str, client_secret: str) -> str: + ... +``` + +## Example with plain python scripts + +When used outside the Orchestrator runtime (e.g. a standalone script), call `wait_for_tracers()` at the end to ensure all traces are flushed before the process exits. ```python hl_lines="3 8" diff --git a/packages/uipath/docs/eval/contains.md b/packages/uipath/docs/eval/contains.md index 73a377742..c8609a0fb 100644 --- a/packages/uipath/docs/eval/contains.md +++ b/packages/uipath/docs/eval/contains.md @@ -18,7 +18,7 @@ The Contains Evaluator checks whether the agent's output contains a specific sea ## Configuration !!! note "Agent Output Structure" - `agent_output` must always be a dictionary. When comparing, the value (or specific field via `target_output_key`) is converted to a string before checking if it contains the search text. + `workload_output` must always be a dictionary. When comparing, the value (or specific field via `target_output_key`) is converted to a string before checking if it contains the search text. ### ContainsEvaluatorConfig @@ -44,13 +44,13 @@ The Contains Evaluator checks whether the agent's output contains a specific sea ```python from uipath.eval.evaluators import ContainsEvaluator -from uipath.eval.models import AgentExecution +from uipath.eval.models import WorkloadExecution # Sample agent execution -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"query": "What is the capital of France?"}, - agent_output={"response": "The capital of France is Paris."}, - agent_trace=[], + workload_output={"response": "The capital of France is Paris."}, + workload_trace=[], ) # Create evaluator - extracts "response" field for comparison @@ -65,7 +65,7 @@ evaluator = ContainsEvaluator( # Evaluate - searches in the "response" field value result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"search_text": "Paris"} ) @@ -76,10 +76,10 @@ print(f"Score: {result.score}") # Output: 1.0 ```python # Sample agent execution -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={}, - agent_output={"message": "Hello World"}, - agent_trace=[], + workload_output={"message": "Hello World"}, + workload_trace=[], ) evaluator = ContainsEvaluator( @@ -93,7 +93,7 @@ evaluator = ContainsEvaluator( # This will fail because of case mismatch result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"search_text": "hello"} ) @@ -106,10 +106,10 @@ Use negation to ensure specific text is NOT present: ```python # Sample agent execution -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={}, - agent_output={"status": "Success: Operation completed"}, - agent_trace=[], + workload_output={"status": "Success: Operation completed"}, + workload_trace=[], ) evaluator = ContainsEvaluator( @@ -123,7 +123,7 @@ evaluator = ContainsEvaluator( # Passes because "error" is NOT found result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"search_text": "error"} ) @@ -134,13 +134,13 @@ print(f"Score: {result.score}") # Output: 1.0 ```python # Sample agent execution -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={}, - agent_output={ + workload_output={ "status": "success", "message": "User profile updated successfully" }, - agent_trace=[], + workload_trace=[], ) evaluator = ContainsEvaluator( @@ -153,7 +153,7 @@ evaluator = ContainsEvaluator( # Only searches within the "message" field result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"search_text": "updated"} ) diff --git a/packages/uipath/docs/eval/custom_evaluators.md b/packages/uipath/docs/eval/custom_evaluators.md index e92fbc2c4..b7b146ba8 100644 --- a/packages/uipath/docs/eval/custom_evaluators.md +++ b/packages/uipath/docs/eval/custom_evaluators.md @@ -94,7 +94,7 @@ Implement the core evaluation logic: ```python from uipath.eval.evaluators import BaseEvaluator from uipath.eval.evaluators.base_evaluator import BaseEvaluatorJustification -from uipath.eval.models import AgentExecution, NumericEvaluationResult +from uipath.eval.models import WorkloadExecution, NumericEvaluationResult class MyCustomEvaluator( BaseEvaluator[MyEvaluationCriteria, MyEvaluatorConfig, BaseEvaluatorJustification] @@ -107,16 +107,16 @@ class MyCustomEvaluator( async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: MyEvaluationCriteria ) -> NumericEvaluationResult: """Evaluate the agent execution against criteria. Args: - agent_execution: The agent execution containing: + workload_execution: The agent execution containing: - agent_input: Input received by the agent - - agent_output: Output produced by the agent - - agent_trace: OpenTelemetry spans with execution trace + - workload_output: Output produced by the agent + - workload_trace: OpenTelemetry spans with execution trace - simulation_instructions: Simulation instructions evaluation_criteria: Criteria to evaluate against @@ -124,7 +124,7 @@ class MyCustomEvaluator( EvaluationResult with score and details """ # Extract data from agent execution - actual_values = self._extract_values(agent_execution) + actual_values = self._extract_values(workload_execution) expected_values = evaluation_criteria.expected_values # Apply case sensitivity from config @@ -146,7 +146,7 @@ class MyCustomEvaluator( }), ) - def _extract_values(self, agent_execution: AgentExecution) -> list[str]: + def _extract_values(self, workload_execution: WorkloadExecution) -> list[str]: """Extract values from agent execution (implement your logic).""" # Your custom extraction logic here return [] @@ -244,9 +244,9 @@ Custom evaluators often need to extract information from tool calls in the agent ```python from uipath.eval._helpers.evaluators_helpers import extract_tool_calls -def _process_tool_calls(self, agent_execution: AgentExecution) -> list[str]: +def _process_tool_calls(self, workload_execution: WorkloadExecution) -> list[str]: """Extract and process tool calls from the execution trace.""" - tool_calls = extract_tool_calls(agent_execution.agent_trace) + tool_calls = extract_tool_calls(workload_execution.workload_trace) results = [] for tool_call in tool_calls: @@ -290,7 +290,7 @@ from uipath.eval.evaluators.base_evaluator import ( BaseEvaluatorJustification, ) from uipath.eval.models import EvaluationResult, NumericEvaluationResult -from uipath.eval.models import AgentExecution +from uipath.eval.models import WorkloadExecution from uipath.eval._helpers.evaluators_helpers import extract_tool_calls @@ -335,13 +335,13 @@ class PatternComparisonEvaluator( async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: PatternEvaluatorCriteria ) -> EvaluationResult: """Evaluate the pattern comparison. Args: - agent_execution: The agent execution containing trace data + workload_execution: The agent execution containing trace data evaluation_criteria: Expected output patterns Returns: @@ -350,7 +350,7 @@ class PatternComparisonEvaluator( expected_output = evaluation_criteria.expected_output # Extract actual output from tool calls - actual_output = self._extract_patterns(agent_execution) + actual_output = self._extract_patterns(workload_execution) # Compute score using intersection over union score = _compute_jaccard_similarity(expected_output, actual_output) @@ -363,17 +363,17 @@ class PatternComparisonEvaluator( }), ) - def _extract_patterns(self, agent_execution: AgentExecution) -> list[str]: + def _extract_patterns(self, workload_execution: WorkloadExecution) -> list[str]: """Extract patterns from tool calls. Args: - agent_execution: The agent execution containing trace data + workload_execution: The agent execution containing trace data Returns: List of pattern strings found """ # Extract tool calls with arguments using the helper function - tool_calls = extract_tool_calls(agent_execution.agent_trace) + tool_calls = extract_tool_calls(workload_execution.workload_trace) for tool_call in tool_calls: if tool_call.name == "DataProcessingTool": @@ -408,13 +408,13 @@ Always include complete type annotations and Google-style docstrings: ```python def _extract_data( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, tool_name: str ) -> list[str]: """Extract data from specific tool calls. Args: - agent_execution: The agent execution to process + workload_execution: The agent execution to process tool_name: The name of the tool to extract data from Returns: @@ -435,13 +435,13 @@ from uipath.eval.models import ErrorEvaluationResult async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: MyCriteria ) -> EvaluationResult: """Evaluate with error handling.""" try: # Your evaluation logic - score = self._compute_score(agent_execution) + score = self._compute_score(workload_execution) return NumericEvaluationResult(score=score) except Exception as e: return ErrorEvaluationResult( @@ -456,12 +456,12 @@ Extract common logic into reusable helper methods: ```python def _extract_from_tool( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, tool_name: str, parameter_name: str ) -> str: """Reusable method to extract parameter from tool calls.""" - tool_calls = extract_tool_calls(agent_execution.agent_trace) + tool_calls = extract_tool_calls(workload_execution.workload_trace) for tool_call in tool_calls: if tool_call.name == tool_name: args = tool_call.args or {} @@ -571,16 +571,16 @@ Test your evaluators locally before registration: ```python import pytest -from uipath.eval.models import AgentExecution +from uipath.eval.models import WorkloadExecution @pytest.mark.asyncio async def test_custom_evaluator() -> None: """Test custom evaluator logic.""" # Create test data - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"query": "test"}, - agent_output={"result": "test output"}, - agent_trace=[], + workload_output={"result": "test output"}, + workload_trace=[], ) # Create evaluator with config @@ -595,7 +595,7 @@ async def test_custom_evaluator() -> None: # Evaluate with criteria criteria = MyEvaluationCriteria(expected_values=["value1"]) - result = await evaluator.evaluate(agent_execution, criteria) + result = await evaluator.evaluate(workload_execution, criteria) # Assert assert result.score >= 0.0 @@ -608,10 +608,10 @@ async def test_custom_evaluator() -> None: ```python def _extract_from_specific_tool( - self, agent_execution: AgentExecution + self, workload_execution: WorkloadExecution ) -> str: """Extract data from a specific tool call.""" - tool_calls = extract_tool_calls(agent_execution.agent_trace) + tool_calls = extract_tool_calls(workload_execution.workload_trace) for tool_call in tool_calls: if tool_call.name == "TargetTool": @@ -643,12 +643,12 @@ def _compute_set_similarity( ```python async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: MyCriteria ) -> EvaluationResult: """Multi-step validation using config settings.""" # Step 1: Validate structure (use strict mode from config) - if not self._validate_structure(agent_execution, self.evaluator_config.strict): + if not self._validate_structure(workload_execution, self.evaluator_config.strict): return NumericEvaluationResult( score=0.0, details=self.validate_justification({ @@ -658,7 +658,7 @@ async def evaluate( ) # Step 2: Extract data - data = self._extract_data(agent_execution) + data = self._extract_data(workload_execution) # Step 3: Compare and score score = self._compare_data(data, evaluation_criteria.expected_data) diff --git a/packages/uipath/docs/eval/exact_match.md b/packages/uipath/docs/eval/exact_match.md index afc11d619..6e5be46b6 100644 --- a/packages/uipath/docs/eval/exact_match.md +++ b/packages/uipath/docs/eval/exact_match.md @@ -18,7 +18,7 @@ The Exact Match Evaluator performs exact string matching between the agent's out ## Configuration !!! note "Agent Output Structure" - `agent_output` must always be a dictionary (e.g., `{"result": "value"}`). To evaluate simple values like strings or numbers, wrap them in a dict and use `target_output_key` to extract the specific field. + `workload_output` must always be a dictionary (e.g., `{"result": "value"}`). To evaluate simple values like strings or numbers, wrap them in a dict and use `target_output_key` to extract the specific field. ### ExactMatchEvaluatorConfig @@ -44,13 +44,13 @@ The Exact Match Evaluator performs exact string matching between the agent's out ```python from uipath.eval.evaluators import ExactMatchEvaluator -from uipath.eval.models import AgentExecution +from uipath.eval.models import WorkloadExecution -# agent_output must be a dict -agent_execution = AgentExecution( +# workload_output must be a dict +workload_execution = WorkloadExecution( agent_input={"query": "What is 2+2?"}, - agent_output={"result": "4"}, - agent_trace=[] + workload_output={"result": "4"}, + workload_trace=[] ) # Create evaluator - extracts "result" field for comparison @@ -65,7 +65,7 @@ evaluator = ExactMatchEvaluator( # Evaluate - compares just the "result" field value result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"expected_output": {"result": "4"}} ) @@ -75,10 +75,10 @@ print(f"Score: {result.score}") # Output: 1.0 ### Case-Sensitive Matching ```python -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={}, - agent_output={"status": "SUCCESS"}, - agent_trace=[] + workload_output={"status": "SUCCESS"}, + workload_trace=[] ) evaluator = ExactMatchEvaluator( @@ -92,7 +92,7 @@ evaluator = ExactMatchEvaluator( # Fails due to case mismatch result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"expected_output": {"status": "success"}} ) @@ -100,7 +100,7 @@ print(f"Score: {result.score}") # Output: 0.0 # This would pass result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"expected_output": {"status": "SUCCESS"}} ) @@ -112,10 +112,10 @@ print(f"Score: {result.score}") # Output: 1.0 When `target_output_key` is `"*"` (default), the entire output dict is compared: ```python -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={}, - agent_output={"status": "success", "code": 200}, - agent_trace=[] + workload_output={"status": "success", "code": 200}, + workload_trace=[] ) evaluator = ExactMatchEvaluator( @@ -128,7 +128,7 @@ evaluator = ExactMatchEvaluator( # Entire dict structure must match result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"expected_output": {"status": "success", "code": 200}} ) @@ -138,13 +138,13 @@ print(f"Score: {result.score}") # Output: 1.0 ### Target Specific Field ```python -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={}, - agent_output={ + workload_output={ "result": "approved", "timestamp": "2024-01-01T12:00:00Z" }, - agent_trace=[] + workload_trace=[] ) evaluator = ExactMatchEvaluator( @@ -157,7 +157,7 @@ evaluator = ExactMatchEvaluator( # Only checks the "result" field result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"expected_output": {"result": "approved"}} ) @@ -167,10 +167,10 @@ print(f"Score: {result.score}") # Output: 1.0 ### Negated Mode ```python -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={}, - agent_output={"result": "error"}, - agent_trace=[] + workload_output={"result": "error"}, + workload_trace=[] ) evaluator = ExactMatchEvaluator( @@ -184,7 +184,7 @@ evaluator = ExactMatchEvaluator( # Passes because outputs do NOT match result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"expected_output": {"result": "success"}} ) @@ -194,10 +194,10 @@ print(f"Score: {result.score}") # Output: 1.0 ### Using Default Criteria ```python -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={}, - agent_output={"status": "OK"}, - agent_trace=[] + workload_output={"status": "OK"}, + workload_trace=[] ) evaluator = ExactMatchEvaluator( @@ -211,7 +211,7 @@ evaluator = ExactMatchEvaluator( # Use default criteria result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, evaluation_criteria=None + workload_execution=workload_execution, evaluation_criteria=None ) print(f"Score: {result.score}") # Output: 1.0 diff --git a/packages/uipath/docs/eval/index.md b/packages/uipath/docs/eval/index.md index 465fcd96a..1848eb51c 100644 --- a/packages/uipath/docs/eval/index.md +++ b/packages/uipath/docs/eval/index.md @@ -72,13 +72,13 @@ To use an evaluator, you typically: ```python from uipath.eval.evaluators import ExactMatchEvaluator -from uipath.eval.models import AgentExecution +from uipath.eval.models import WorkloadExecution # Sample agent execution (this should be replaced with your agent run data) -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"query": "Greet the world"}, - agent_output={"result": "hello, world!"}, - agent_trace=[], + workload_output={"result": "hello, world!"}, + workload_trace=[], ) # Create evaluator @@ -93,7 +93,7 @@ evaluator = ExactMatchEvaluator( # Evaluate result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"expected_output": {"result": "Hello, World!"}} ) diff --git a/packages/uipath/docs/eval/json_similarity.md b/packages/uipath/docs/eval/json_similarity.md index 19bf60739..006b21a0e 100644 --- a/packages/uipath/docs/eval/json_similarity.md +++ b/packages/uipath/docs/eval/json_similarity.md @@ -18,7 +18,7 @@ The JSON Similarity Evaluator performs flexible structural comparison of JSON-li ## Configuration !!! note "Agent Output Structure" - `agent_output` must always be a dictionary. The evaluator compares dictionary structures recursively, making it ideal for complex nested JSON-like outputs. + `workload_output` must always be a dictionary. The evaluator compares dictionary structures recursively, making it ideal for complex nested JSON-like outputs. ### JsonSimilarityEvaluatorConfig @@ -59,12 +59,12 @@ The final score represents the percentage of matching leaf nodes in the tree str ```python from uipath.eval.evaluators import JsonSimilarityEvaluator -from uipath.eval.models import AgentExecution +from uipath.eval.models import WorkloadExecution -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={}, - agent_output={"name": "John Doe", "age": 30, "city": "New York"}, - agent_trace=[] + workload_output={"name": "John Doe", "age": 30, "city": "New York"}, + workload_trace=[] ) evaluator = JsonSimilarityEvaluator( @@ -76,7 +76,7 @@ evaluator = JsonSimilarityEvaluator( ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_output": {"name": "John Doe", "age": 30, "city": "New York"} } @@ -90,10 +90,10 @@ print(f"Total: {result.details.total_leaves}") # 3.0 ### Numeric Tolerance ```python -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={}, - agent_output={"temperature": 20.5, "humidity": 65}, - agent_trace=[] + workload_output={"temperature": 20.5, "humidity": 65}, + workload_trace=[] ) evaluator = JsonSimilarityEvaluator( @@ -103,7 +103,7 @@ evaluator = JsonSimilarityEvaluator( # Slightly different numbers result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_output": {"temperature": 20.3, "humidity": 65} } @@ -116,10 +116,10 @@ print(f"Score: {result.score}") # ~0.99 (very high similarity) ### String Similarity ```python -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={}, - agent_output={"status": "completed successfully"}, - agent_trace=[] + workload_output={"status": "completed successfully"}, + workload_trace=[] ) evaluator = JsonSimilarityEvaluator( @@ -129,7 +129,7 @@ evaluator = JsonSimilarityEvaluator( # Similar but not exact string result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_output": {"status": "completed sucessfully"} # typo } @@ -142,9 +142,9 @@ print(f"Score: {result.score}") # ~0.95 (high similarity despite typo) ### Nested Structures ```python -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={}, - agent_output={ + workload_output={ "user": { "name": "Alice", "profile": { @@ -154,7 +154,7 @@ agent_execution = AgentExecution( }, "status": "active" }, - agent_trace=[] + workload_trace=[] ) evaluator = JsonSimilarityEvaluator( @@ -163,7 +163,7 @@ evaluator = JsonSimilarityEvaluator( ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_output": { "user": { @@ -184,10 +184,10 @@ print(f"Score: {result.score}") # Output: 1.0 ### Array Comparison ```python -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={}, - agent_output={"items": ["apple", "banana", "orange"]}, - agent_trace=[] + workload_output={"items": ["apple", "banana", "orange"]}, + workload_trace=[] ) evaluator = JsonSimilarityEvaluator( @@ -197,7 +197,7 @@ evaluator = JsonSimilarityEvaluator( # Partial match (2 out of 3 correct) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_output": {"items": ["apple", "banana", "grape"]} } @@ -209,14 +209,14 @@ print(f"Score: {result.score}") # ~0.67 (2/3 correct) ### Handling Extra Keys in Actual Output ```python -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={}, - agent_output={ + workload_output={ "name": "Bob", "age": 30, "extra_field": "ignored" # Extra field in actual output }, - agent_trace=[] + workload_trace=[] ) evaluator = JsonSimilarityEvaluator( @@ -226,7 +226,7 @@ evaluator = JsonSimilarityEvaluator( # Only expected keys are evaluated result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_output": { "name": "Bob", @@ -241,13 +241,13 @@ print(f"Score: {result.score}") # Output: 1.0 (extra fields ignored) ### Target Specific Field ```python -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={}, - agent_output={ + workload_output={ "result": {"score": 95, "passed": True}, "metadata": {"timestamp": "2024-01-01"} }, - agent_trace=[] + workload_trace=[] ) evaluator = JsonSimilarityEvaluator( @@ -260,7 +260,7 @@ evaluator = JsonSimilarityEvaluator( # Only compares the "result" field result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_output": {"result": {"score": 95, "passed": True}} } diff --git a/packages/uipath/docs/eval/llm_judge_output.md b/packages/uipath/docs/eval/llm_judge_output.md index c987a835c..d555e4d1e 100644 --- a/packages/uipath/docs/eval/llm_judge_output.md +++ b/packages/uipath/docs/eval/llm_judge_output.md @@ -109,12 +109,12 @@ The prompt template supports these placeholders: ```python from uipath.eval.evaluators import LLMJudgeOutputEvaluator -from uipath.eval.models import AgentExecution +from uipath.eval.models import WorkloadExecution -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"query": "What is the capital of France?"}, - agent_output={"answer": "Paris is the capital city of France."}, - agent_trace=[] + workload_output={"answer": "Paris is the capital city of France."}, + workload_trace=[] ) evaluator = LLMJudgeOutputEvaluator( @@ -129,7 +129,7 @@ evaluator = LLMJudgeOutputEvaluator( ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_output": {"answer": "The capital of France is Paris."} } @@ -152,10 +152,10 @@ Expected Output: {{ExpectedOutput}} Provide a score from 0-100 based on semantic similarity. """ -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={}, - agent_output={"message": "The product has been successfully added to your cart."}, - agent_trace=[] + workload_output={"message": "The product has been successfully added to your cart."}, + workload_trace=[] ) evaluator = LLMJudgeOutputEvaluator( @@ -170,7 +170,7 @@ evaluator = LLMJudgeOutputEvaluator( ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_output": {"message": "Item added to shopping cart."} } @@ -183,9 +183,9 @@ print(f"Justification: {result.details}") #### Evaluating Natural Language Quality ```python -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"task": "Write a professional email"}, - agent_output={"email": """Dear Customer, + workload_output={"email": """Dear Customer, Thank you for your inquiry. We have reviewed your request and are pleased to inform you that we can accommodate your @@ -193,7 +193,7 @@ needs. Please let us know if you have any questions. Best regards, Support Team"""}, - agent_trace=[] + workload_trace=[] ) evaluator = LLMJudgeOutputEvaluator( @@ -207,7 +207,7 @@ evaluator = LLMJudgeOutputEvaluator( ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_output": {"email": "A professional, courteous response addressing the customer's inquiry"} } @@ -271,19 +271,19 @@ evaluator = LLMJudgeStrictJSONSimilarityOutputEvaluator( } ) -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={}, - agent_output={ + workload_output={ "status": "success", "user_id": 12345, "name": "John Doe", "email": "john@example.com" }, - agent_trace=[] + workload_trace=[] ) result = await evaluator.evaluate( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_output": { "status": "success", diff --git a/packages/uipath/docs/eval/llm_judge_trajectory.md b/packages/uipath/docs/eval/llm_judge_trajectory.md index 8705f1a29..d5846e387 100644 --- a/packages/uipath/docs/eval/llm_judge_trajectory.md +++ b/packages/uipath/docs/eval/llm_judge_trajectory.md @@ -111,12 +111,12 @@ The prompt template supports these placeholders: ```python from uipath.eval.evaluators import LLMJudgeTrajectoryEvaluator -from uipath.eval.models import AgentExecution +from uipath.eval.models import WorkloadExecution -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"user_query": "Book a flight to Paris"}, - agent_output={"booking_id": "FL123", "status": "confirmed"}, - agent_trace=[ + workload_output={"booking_id": "FL123", "status": "confirmed"}, + workload_trace=[ # Trace contains spans showing the agent's execution path # Each span represents a step in the agent's decision-making ] @@ -133,7 +133,7 @@ evaluator = LLMJudgeTrajectoryEvaluator( ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_agent_behavior": """ The agent should: @@ -152,10 +152,10 @@ print(f"Justification: {result.details}") #### Validating Tool Usage Sequence ```python -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"task": "Update user profile and send notification"}, - agent_output={"status": "completed"}, - agent_trace=[ + workload_output={"status": "completed"}, + workload_trace=[ # Spans showing: validate_user -> update_profile -> send_notification ] ) @@ -170,7 +170,7 @@ evaluator = LLMJudgeTrajectoryEvaluator( ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_agent_behavior": """ The agent must: @@ -259,10 +259,10 @@ Same as `LLMJudgeTrajectoryEvaluatorConfig` but with: ```python from uipath.eval.evaluators import LLMJudgeTrajectorySimulationEvaluator -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"query": "Book a flight to Paris for tomorrow"}, - agent_output={"booking_id": "FL123", "status": "confirmed"}, - agent_trace=[ + workload_output={"booking_id": "FL123", "status": "confirmed"}, + workload_trace=[ # Execution spans showing tool calls and their simulated responses ], simulation_instructions=""" @@ -284,7 +284,7 @@ evaluator = LLMJudgeTrajectorySimulationEvaluator( ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_agent_behavior": """ The agent should: @@ -303,7 +303,7 @@ print(f"Justification: {result.details}") ## Understanding Agent Traces -The `agent_trace` contains execution spans that show: +The `workload_trace` contains execution spans that show: - Tool calls made by the agent - LLM reasoning steps @@ -313,7 +313,7 @@ The `agent_trace` contains execution spans that show: Example trace structure: ```python -agent_trace = [ +workload_trace = [ { "name": "search_flights", "type": "tool", diff --git a/packages/uipath/docs/eval/tool_call_args.md b/packages/uipath/docs/eval/tool_call_args.md index ced1c63fd..94705d25b 100644 --- a/packages/uipath/docs/eval/tool_call_args.md +++ b/packages/uipath/docs/eval/tool_call_args.md @@ -69,7 +69,7 @@ For each tool call: ```python from opentelemetry.sdk.trace import ReadableSpan from uipath.eval.evaluators import ToolCallArgsEvaluator -from uipath.eval.models import AgentExecution +from uipath.eval.models import WorkloadExecution # Sample agent execution with tool calls and arguments mock_spans = [ @@ -84,10 +84,10 @@ mock_spans = [ ), ] -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"user_id": 123, "action": "update"}, - agent_output={"status": "success"}, - agent_trace=mock_spans, + workload_output={"status": "success"}, + workload_trace=mock_spans, ) evaluator = ToolCallArgsEvaluator( @@ -100,7 +100,7 @@ evaluator = ToolCallArgsEvaluator( ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls": [ { @@ -136,10 +136,10 @@ mock_spans = [ ), ] -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"action": "fetch_users"}, - agent_output={"status": "success"}, - agent_trace=mock_spans, + workload_output={"status": "success"}, + workload_trace=mock_spans, ) evaluator = ToolCallArgsEvaluator( @@ -153,7 +153,7 @@ evaluator = ToolCallArgsEvaluator( # Arguments must match exactly result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls": [ { @@ -177,7 +177,7 @@ print(f"Score: {result.score}") # 1.0 ```python from opentelemetry.sdk.trace import ReadableSpan from uipath.eval.evaluators import ToolCallArgsEvaluator -from uipath.eval.models import AgentExecution +from uipath.eval.models import WorkloadExecution # Agent called 3 tools, but only 2 match the expected args mock_spans = [ @@ -210,10 +210,10 @@ mock_spans = [ ), ] -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"task": "Update user profile"}, - agent_output={"status": "updated"}, - agent_trace=mock_spans, + workload_output={"status": "updated"}, + workload_trace=mock_spans, ) evaluator = ToolCallArgsEvaluator( @@ -226,7 +226,7 @@ evaluator = ToolCallArgsEvaluator( ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls": [ { @@ -270,10 +270,10 @@ mock_spans = [ ), ] -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"action": "send_welcome"}, - agent_output={"status": "sent"}, - agent_trace=mock_spans, + workload_output={"status": "sent"}, + workload_trace=mock_spans, ) evaluator = ToolCallArgsEvaluator( @@ -287,7 +287,7 @@ evaluator = ToolCallArgsEvaluator( # Only validate specific arguments, allow extras result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls": [ { @@ -341,10 +341,10 @@ mock_spans = [ ), ] -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"task": "Update user profile"}, - agent_output={"status": "updated"}, - agent_trace=mock_spans, + workload_output={"status": "updated"}, + workload_trace=mock_spans, ) evaluator = ToolCallArgsEvaluator( @@ -357,7 +357,7 @@ evaluator = ToolCallArgsEvaluator( ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls": [ { @@ -399,10 +399,10 @@ mock_spans = [ ), ] -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"task": "Create order"}, - agent_output={"status": "created"}, - agent_trace=mock_spans, + workload_output={"status": "created"}, + workload_trace=mock_spans, ) evaluator = ToolCallArgsEvaluator( @@ -416,7 +416,7 @@ evaluator = ToolCallArgsEvaluator( # Validate complex nested structures result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls": [ { diff --git a/packages/uipath/docs/eval/tool_call_count.md b/packages/uipath/docs/eval/tool_call_count.md index e72e30c2d..a07f15dd6 100644 --- a/packages/uipath/docs/eval/tool_call_count.md +++ b/packages/uipath/docs/eval/tool_call_count.md @@ -72,7 +72,7 @@ Each tool is evaluated independently: ```python from opentelemetry.sdk.trace import ReadableSpan from uipath.eval.evaluators import ToolCallCountEvaluator -from uipath.eval.models import AgentExecution +from uipath.eval.models import WorkloadExecution # Sample agent execution with tool calls mock_spans = [ @@ -92,10 +92,10 @@ mock_spans = [ attributes={"tool.name": "send_notification"}), ] -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"task": "Fetch and process data"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = ToolCallCountEvaluator( @@ -107,7 +107,7 @@ evaluator = ToolCallCountEvaluator( ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls_count": { "fetch_data": ("=", 1), # Called exactly once @@ -154,10 +154,10 @@ mock_spans.append( ) ) -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"task": "Fetch and process data"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = ToolCallCountEvaluator( @@ -169,7 +169,7 @@ evaluator = ToolCallCountEvaluator( ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls_count": { "fetch_data": ("=", 1), # ✓ Matches (1 call) @@ -216,10 +216,10 @@ mock_spans = [ ), ] -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"task": "Database operation"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = ToolCallCountEvaluator( @@ -231,7 +231,7 @@ evaluator = ToolCallCountEvaluator( ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls_count": { "authenticate": ("=", 1), # ✓ Matches (1 call) @@ -278,10 +278,10 @@ mock_spans = [ ), ] -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"task": "Optimize resource usage"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = ToolCallCountEvaluator( @@ -294,7 +294,7 @@ evaluator = ToolCallCountEvaluator( # Ensure expensive operations aren't called too many times result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls_count": { "expensive_api_call": ("<=", 1), # Should not be called more than once @@ -336,10 +336,10 @@ for i in range(10): ), ]) -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"task": "Process 10 items"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = ToolCallCountEvaluator( @@ -352,7 +352,7 @@ evaluator = ToolCallCountEvaluator( # Verify loop processed correct number of items result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls_count": { "process_item": ("=", 10), # Should process 10 items @@ -398,10 +398,10 @@ mock_spans = [ ), ] -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"task": "Retry operation"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = ToolCallCountEvaluator( @@ -414,7 +414,7 @@ evaluator = ToolCallCountEvaluator( # Verify retry logic doesn't exceed limits result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls_count": { "attempt_operation": ("<=", 3), # Max 3 retries @@ -454,10 +454,10 @@ mock_spans = [ ), ] -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"task": "Secure operation"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = ToolCallCountEvaluator( @@ -470,7 +470,7 @@ evaluator = ToolCallCountEvaluator( # Ensure agent calls important tools result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls_count": { "validate_input": (">=", 1), # Must validate at least once diff --git a/packages/uipath/docs/eval/tool_call_order.md b/packages/uipath/docs/eval/tool_call_order.md index 886f8be41..2a703f572 100644 --- a/packages/uipath/docs/eval/tool_call_order.md +++ b/packages/uipath/docs/eval/tool_call_order.md @@ -68,7 +68,7 @@ Where LCS is the Longest Common Subsequence between expected and actual tool cal ```python from opentelemetry.sdk.trace import ReadableSpan from uipath.eval.evaluators import ToolCallOrderEvaluator -from uipath.eval.models import AgentExecution +from uipath.eval.models import WorkloadExecution # Create mock spans representing tool calls in execution trace mock_spans = [ @@ -92,10 +92,10 @@ mock_spans = [ ), ] -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"task": "Process user order"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = ToolCallOrderEvaluator( @@ -107,7 +107,7 @@ evaluator = ToolCallOrderEvaluator( ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls_order": [ "validate_user", @@ -149,10 +149,10 @@ mock_spans = [ ), ] -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"task": "Access secured resource"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = ToolCallOrderEvaluator( @@ -164,7 +164,7 @@ evaluator = ToolCallOrderEvaluator( ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls_order": [ "authenticate_user", @@ -205,10 +205,10 @@ mock_spans = [ ), ] -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"task": "Search and display"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = ToolCallOrderEvaluator( @@ -223,7 +223,7 @@ evaluator = ToolCallOrderEvaluator( expected = ["search", "filter", "sort", "display"] result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls_order": expected } @@ -266,10 +266,10 @@ mock_spans = [ ), ] -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"task": "Update database"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = ToolCallOrderEvaluator( @@ -281,7 +281,7 @@ evaluator = ToolCallOrderEvaluator( ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls_order": [ "begin_transaction", @@ -334,10 +334,10 @@ mock_spans = [ ), ] -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"task": "API integration"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = ToolCallOrderEvaluator( @@ -349,7 +349,7 @@ evaluator = ToolCallOrderEvaluator( ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls_order": [ "get_api_token", @@ -390,10 +390,10 @@ mock_spans = [ ), ] -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"task": "Standard workflow"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = ToolCallOrderEvaluator( @@ -409,7 +409,7 @@ evaluator = ToolCallOrderEvaluator( # Use default criteria result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria=None # Uses default ) diff --git a/packages/uipath/docs/eval/tool_call_output.md b/packages/uipath/docs/eval/tool_call_output.md index 9b3cd29f2..7938dc9db 100644 --- a/packages/uipath/docs/eval/tool_call_output.md +++ b/packages/uipath/docs/eval/tool_call_output.md @@ -64,7 +64,7 @@ For each expected tool output: ```python from opentelemetry.sdk.trace import ReadableSpan from uipath.eval.evaluators import ToolCallOutputEvaluator -from uipath.eval.models import AgentExecution +from uipath.eval.models import WorkloadExecution # Sample agent execution with tool calls and outputs mock_spans = [ @@ -79,10 +79,10 @@ mock_spans = [ ), ] -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"user_id": 123}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = ToolCallOutputEvaluator( @@ -94,7 +94,7 @@ evaluator = ToolCallOutputEvaluator( ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_outputs": [ { @@ -126,10 +126,10 @@ mock_spans = [ ), ] -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"items": ["item1", "item2"]}, - agent_output={"status": "calculated"}, - agent_trace=mock_spans, + workload_output={"status": "calculated"}, + workload_trace=mock_spans, ) evaluator = ToolCallOutputEvaluator( @@ -142,7 +142,7 @@ evaluator = ToolCallOutputEvaluator( # Outputs must match exactly result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_outputs": [ { @@ -162,7 +162,7 @@ print(f"Score: {result.score}") # 1.0 ```python from opentelemetry.sdk.trace import ReadableSpan from uipath.eval.evaluators import ToolCallOutputEvaluator -from uipath.eval.models import AgentExecution +from uipath.eval.models import WorkloadExecution # Agent produced 3 outputs, but only 2 match expected mock_spans = [ @@ -195,10 +195,10 @@ mock_spans = [ ), ] -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"task": "Process data pipeline"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = ToolCallOutputEvaluator( @@ -210,7 +210,7 @@ evaluator = ToolCallOutputEvaluator( ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_outputs": [ { @@ -269,10 +269,10 @@ mock_spans = [ ), ] -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"task": "Process data pipeline"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = ToolCallOutputEvaluator( @@ -284,7 +284,7 @@ evaluator = ToolCallOutputEvaluator( ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_outputs": [ { @@ -323,10 +323,10 @@ mock_spans = [ ), ] -agent_execution = AgentExecution( +workload_execution = WorkloadExecution( agent_input={"task": "Generate report"}, - agent_output={"status": "generated"}, - agent_trace=mock_spans, + workload_output={"status": "generated"}, + workload_trace=mock_spans, ) evaluator = ToolCallOutputEvaluator( @@ -338,7 +338,7 @@ evaluator = ToolCallOutputEvaluator( ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_outputs": [ { diff --git a/packages/uipath/docs/index.md b/packages/uipath/docs/index.md index 231add0ca..1a88e6b2a 100644 --- a/packages/uipath/docs/index.md +++ b/packages/uipath/docs/index.md @@ -2,38 +2,32 @@ title: Getting Started --- -
-- __🚨 Breaking changes__ - - --- - - UiPath Python SDK v2.2.0+ will introduce **breaking changes** starting **November 26, 2025** - [See Details](./core/release_notes.md) -
+

What do you want to build?

-- __UiPath SDK__ +- __Python Coded Functions__ --- - Code with full UiPath context to build custom automations and agents from the ground up. + Deterministic Python automation with typed input/output. No LLM required. Runs as an Orchestrator job, invokable from Maestro, Studio, or the CLI. - [Start Building](./core/getting_started.md) + **Requires:** `uipath` -
+ [Build a Function](./core/functions.md) -
-- __UiPath MCP SDK__ +- __Python Coded Agents__ --- - Build and host Coded MCP Servers within UiPath. + AI-driven automation with LLM reasoning loops. Uses the `uipath` SDK for platform services plus a framework extension of your choice. - [Start Building](./mcp/quick_start.md) + **Requires:** `uipath` + one of the extensions below + + [Build an Agent](./core/agents.md)
-

Extensions

+

Agent Framework Extensions

- __UiPath Langchain SDK__ @@ -60,3 +54,15 @@ title: Getting Started [Get Started](./openai-agents/quick_start.md)
+ +

Other SDKs

+
+- __UiPath MCP SDK__ + + --- + + Build and host Coded MCP Servers within UiPath. + + [Start Building](./mcp/quick_start.md) + +
diff --git a/packages/uipath/mkdocs.yml b/packages/uipath/mkdocs.yml index 0b7fafd34..382e0fdd5 100644 --- a/packages/uipath/mkdocs.yml +++ b/packages/uipath/mkdocs.yml @@ -56,11 +56,13 @@ nav: - Home: index.md - UiPath SDK: - Getting Started: core/getting_started.md - - Release Notes: core/release_notes.md - - Environment Variables: core/environment_variables.md + - Python Coded Functions: core/functions.md + - Python Coded Agents: core/agents.md - CLI Reference: cli/index.md - Tracing: core/traced.md - Studio Web Integration: core/studio_web.md + - Environment Variables: core/environment_variables.md + - Release Notes: core/release_notes.md - Services: - Assets: core/assets.md - Attachments: core/attachments.md @@ -101,6 +103,7 @@ nav: - Getting Started: langchain/quick_start.md - Chat Models: langchain/chat_models.md - Context Grounding: langchain/context_grounding.md + - Guardrails: langchain/guardrails.md - Human In The Loop: langchain/human_in_the_loop.md - Sample Agents: https://github.com/UiPath/uipath-langchain-python/tree/main/samples - UiPath LlamaIndex SDK: @@ -120,10 +123,12 @@ nav: plugins: - search - llmstxt: + full_output: llms-full.txt sections: "UiPath SDK": - core/*.md - cli/*.md + - eval/*.md "UiPath MCP SDK": - mcp/*.md "UiPath LangChain SDK": diff --git a/packages/uipath/pyproject.toml b/packages/uipath/pyproject.toml index 49a172d07..599b65f33 100644 --- a/packages/uipath/pyproject.toml +++ b/packages/uipath/pyproject.toml @@ -1,13 +1,13 @@ [project] name = "uipath" -version = "2.10.40" +version = "2.14.1" description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" dependencies = [ - "uipath-core>=0.5.8, <0.6.0", - "uipath-runtime>=0.10.0, <0.11.0", - "uipath-platform>=0.1.13, <0.2.0", + "uipath-core>=0.5.30, <0.6.0", + "uipath-runtime>=0.13.0, <0.14.0", + "uipath-platform>=0.2.14, <0.3.0", "click>=8.3.1", "httpx>=0.28.1", "pyjwt>=2.10.1", @@ -24,6 +24,7 @@ dependencies = [ "mermaid-builder==0.0.3", "graphtty==0.1.8", "applicationinsights>=0.11.10", + "pyyaml>=6.0, <7.0", ] classifiers = [ "Intended Audience :: Developers", @@ -45,6 +46,9 @@ Documentation = "https://uipath.github.io/uipath-python/" [project.scripts] uipath = "uipath._cli:cli" +[project.optional-dependencies] +ipc = ["uipath-ipc>=2.5.1,<2.6.0"] + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" @@ -75,7 +79,9 @@ dev = [ "mkdocs-llmstxt>=0.5.0", "inflection>=0.5.1", "types-toml>=0.10.8", + "types-PyYAML>=6.0", "pytest-timeout>=2.4.0", + "uipath-ipc>=2.5.1,<2.6.0", ] [tool.hatch.build.targets.wheel] @@ -137,15 +143,39 @@ warn_required_dynamic_aliases = true [tool.pytest.ini_options] testpaths = ["tests"] python_files = "test_*.py" -addopts = "-ra -q --cov" +addopts = "-ra -q --cov=src --cov-report=term-missing" asyncio_default_fixture_loop_scope = "function" asyncio_mode = "auto" +[tool.coverage.run] +source = ["src"] +relative_files = true +omit = [ + "*/tests/*", + "*/__pycache__/*", + "*/site-packages/*", + "*/conftest.py", +] + [tool.coverage.report] show_missing = true +precision = 2 +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "raise NotImplementedError", + "if TYPE_CHECKING:", + "@(abc\\.)?abstractmethod", +] -[tool.coverage.run] -source = ["src"] +[tool.uv] +exclude-newer = "2 days" + +[tool.uv.exclude-newer-package] +uipath-core = false +uipath-runtime = false +uipath-platform = false +uipath-ipc = false [tool.uv.sources] uipath-core = { path = "../uipath-core", editable = true } diff --git a/packages/uipath/samples/attachment_evaluation_test/README.md b/packages/uipath/samples/attachment_evaluation_test/README.md index ab6a35366..d5c30e8d3 100644 --- a/packages/uipath/samples/attachment_evaluation_test/README.md +++ b/packages/uipath/samples/attachment_evaluation_test/README.md @@ -159,7 +159,7 @@ The `targetOutputKey` points to the field containing the attachment URI. ### 3. Automatic Download and Evaluation When the evaluator runs: -1. Extracts value from `agent_output["report"]` +1. Extracts value from `workload_output["report"]` 2. Detects it's an attachment URI (matches pattern) 3. Downloads the attachment content automatically 4. Evaluates the downloaded content against expected criteria @@ -201,14 +201,14 @@ The `targetOutputKey` in evaluator configuration specifies which field contains ```json { - "targetOutputKey": "report" // Looks for agent_output["report"] + "targetOutputKey": "report" // Looks for workload_output["report"] } ``` For nested paths, use dot notation: ```json { - "targetOutputKey": "results.report" // agent_output["results"]["report"] + "targetOutputKey": "results.report" // workload_output["results"]["report"] } ``` diff --git a/packages/uipath/samples/calculator/evaluations/evaluators/custom/correct_operator.py b/packages/uipath/samples/calculator/evaluations/evaluators/custom/correct_operator.py index 0e79f3445..c82797ce1 100644 --- a/packages/uipath/samples/calculator/evaluations/evaluators/custom/correct_operator.py +++ b/packages/uipath/samples/calculator/evaluations/evaluators/custom/correct_operator.py @@ -8,9 +8,9 @@ BaseEvaluatorJustification, ) from uipath.eval.models import ( - AgentExecution, EvaluationResult, NumericEvaluationResult, + WorkloadExecution, ) @@ -41,8 +41,8 @@ class CorrectOperatorEvaluator( ): """A custom evaluator that checks if the correct operator is being used by the agent""" - def extract_operator_from_spans(self, agent_trace: list[ReadableSpan]) -> str: - for span in agent_trace: + def extract_operator_from_spans(self, workload_trace: list[ReadableSpan]) -> str: + for span in workload_trace: if span.name == "track_operator": if span.attributes: input_value_as_str = span.attributes.get("input.value", "{}") @@ -57,10 +57,10 @@ def get_evaluator_id(cls) -> str: async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: CorrectOperatorEvaluationCriteria, ) -> EvaluationResult: - actual_operator = self.extract_operator_from_spans(agent_execution.agent_trace) + actual_operator = self.extract_operator_from_spans(workload_execution.workload_trace) print(actual_operator) is_expected_operator = evaluation_criteria.operator == actual_operator if self.evaluator_config.negated: diff --git a/packages/uipath/samples/classification_agent/evaluations/evaluators/custom/balanced_accuracy_evaluator.py b/packages/uipath/samples/classification_agent/evaluations/evaluators/custom/balanced_accuracy_evaluator.py index 3ef7fd806..043df839c 100644 --- a/packages/uipath/samples/classification_agent/evaluations/evaluators/custom/balanced_accuracy_evaluator.py +++ b/packages/uipath/samples/classification_agent/evaluations/evaluators/custom/balanced_accuracy_evaluator.py @@ -19,9 +19,9 @@ OutputEvaluatorConfig, ) from uipath.eval.models import ( - AgentExecution, EvaluationResult, NumericEvaluationResult, + WorkloadExecution, ) from uipath.eval.models.models import ( EvaluationResultDto, @@ -80,10 +80,10 @@ def reduce_scores(results: list[EvaluationResultDto]) -> float: async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: BalancedAccuracyEvaluationCriteria, ) -> EvaluationResult: - predicted_class = str(self._get_actual_output(agent_execution)).lower() + predicted_class = str(self._get_actual_output(workload_execution)).lower() expected_class = evaluation_criteria.expected_class.lower() classes = [c.lower() for c in self.evaluator_config.classes] class_counts = { diff --git a/packages/uipath/samples/csv-processor/evaluations/evaluators/custom/attachment_created_evaluator.py b/packages/uipath/samples/csv-processor/evaluations/evaluators/custom/attachment_created_evaluator.py index 0b46e77b9..a6627b984 100644 --- a/packages/uipath/samples/csv-processor/evaluations/evaluators/custom/attachment_created_evaluator.py +++ b/packages/uipath/samples/csv-processor/evaluations/evaluators/custom/attachment_created_evaluator.py @@ -4,7 +4,11 @@ BaseEvaluatorConfig, BaseEvaluatorJustification, ) -from uipath.eval.models import AgentExecution, EvaluationResult, NumericEvaluationResult +from uipath.eval.models import ( + EvaluationResult, + NumericEvaluationResult, + WorkloadExecution, +) class AttachmentCreatedEvaluationCriteria(BaseEvaluationCriteria): @@ -39,7 +43,7 @@ def get_evaluator_id(cls) -> str: async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: AttachmentCreatedEvaluationCriteria, ) -> EvaluationResult: # Check if the agent created an attachment by looking for: @@ -49,7 +53,7 @@ async def evaluate( attachment_created = False # Look for attachment creation in traces - for span in agent_execution.agent_trace: + for span in workload_execution.workload_trace: # Check span name for attachment operations if "attachment" in span.name.lower() or "create" in span.name.lower(): attachment_created = True @@ -70,8 +74,8 @@ async def evaluate( break # Also check if output contains attachment information - if not attachment_created and agent_execution.agent_output: - output_str = str(agent_execution.agent_output) + if not attachment_created and workload_execution.workload_output: + output_str = str(workload_execution.workload_output) if ( "attachment" in output_str.lower() or evaluation_criteria.attachment_name in output_str diff --git a/packages/uipath/samples/csv-processor/evaluations/evaluators/custom/csv_columns_evaluator.py b/packages/uipath/samples/csv-processor/evaluations/evaluators/custom/csv_columns_evaluator.py index 29efe1ce7..dd9dc40fb 100644 --- a/packages/uipath/samples/csv-processor/evaluations/evaluators/custom/csv_columns_evaluator.py +++ b/packages/uipath/samples/csv-processor/evaluations/evaluators/custom/csv_columns_evaluator.py @@ -6,7 +6,11 @@ BaseEvaluatorConfig, BaseEvaluatorJustification, ) -from uipath.eval.models import AgentExecution, EvaluationResult, NumericEvaluationResult +from uipath.eval.models import ( + EvaluationResult, + NumericEvaluationResult, + WorkloadExecution, +) class CSVColumnsEvaluationCriteria(BaseEvaluationCriteria): @@ -39,7 +43,7 @@ def get_evaluator_id(cls) -> str: async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: CSVColumnsEvaluationCriteria, ) -> EvaluationResult: # Check if all expected columns are mentioned in the output @@ -55,7 +59,7 @@ async def evaluate( ) # Look for column names in agent traces (where print output is captured) - for span in agent_execution.agent_trace: + for span in workload_execution.workload_trace: # Check span attributes if span.attributes: for attr_value in span.attributes.values(): @@ -75,8 +79,8 @@ async def evaluate( columns_found.add(column) # Also check in the output - if len(columns_found) < total_columns and agent_execution.agent_output: - output_str = str(agent_execution.agent_output) + if len(columns_found) < total_columns and workload_execution.workload_output: + output_str = str(workload_execution.workload_output) for column in evaluation_criteria.expected_columns: if column in output_str: columns_found.add(column) diff --git a/packages/uipath/samples/csv-processor/evaluations/evaluators/custom/csv_shape_evaluator.py b/packages/uipath/samples/csv-processor/evaluations/evaluators/custom/csv_shape_evaluator.py index deb2b83f4..1d50827ee 100644 --- a/packages/uipath/samples/csv-processor/evaluations/evaluators/custom/csv_shape_evaluator.py +++ b/packages/uipath/samples/csv-processor/evaluations/evaluators/custom/csv_shape_evaluator.py @@ -4,7 +4,11 @@ BaseEvaluatorConfig, BaseEvaluatorJustification, ) -from uipath.eval.models import AgentExecution, EvaluationResult, NumericEvaluationResult +from uipath.eval.models import ( + EvaluationResult, + NumericEvaluationResult, + WorkloadExecution, +) class CSVShapeEvaluationCriteria(BaseEvaluationCriteria): @@ -38,7 +42,7 @@ def get_evaluator_id(cls) -> str: async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: CSVShapeEvaluationCriteria, ) -> EvaluationResult: # The agent prints: "CSV shape (rows, columns)\nCSV columns [...]" @@ -48,7 +52,7 @@ async def evaluate( shape_found = False # Check agent traces (where print output is captured) - for span in agent_execution.agent_trace: + for span in workload_execution.workload_trace: # Check span attributes if span.attributes: for attr_value in span.attributes.values(): @@ -72,8 +76,8 @@ async def evaluate( break # Check agent output - if not shape_found and agent_execution.agent_output: - output_str = str(agent_execution.agent_output) + if not shape_found and workload_execution.workload_output: + output_str = str(workload_execution.workload_output) shape_found = expected_shape in output_str return NumericEvaluationResult( diff --git a/packages/uipath/samples/list-mcp-agent/main.py b/packages/uipath/samples/list-mcp-agent/main.py index 2d71c7697..d1c78b989 100644 --- a/packages/uipath/samples/list-mcp-agent/main.py +++ b/packages/uipath/samples/list-mcp-agent/main.py @@ -24,9 +24,9 @@ def list_mcp_servers() -> list[McpServer]: return uipath.mcp.list(folder_path="Shared") -def retrieve_mcp_server(slug: str) -> McpServer: +def retrieve_mcp_server(name: str) -> McpServer: uipath = UiPath() - return uipath.mcp.retrieve(slug, folder_path="Shared") + return uipath.mcp.retrieve(name, folder_path="Shared") async def connect_and_list_tools(server: McpServer) -> list[str]: diff --git a/packages/uipath/samples/list_target_output_key_test/bindings.json b/packages/uipath/samples/list_target_output_key_test/bindings.json new file mode 100644 index 000000000..6122d0e77 --- /dev/null +++ b/packages/uipath/samples/list_target_output_key_test/bindings.json @@ -0,0 +1,4 @@ +{ + "version": "2.0", + "resources": [] +} diff --git a/packages/uipath/samples/list_target_output_key_test/entry-points.json b/packages/uipath/samples/list_target_output_key_test/entry-points.json new file mode 100644 index 000000000..55ce49fba --- /dev/null +++ b/packages/uipath/samples/list_target_output_key_test/entry-points.json @@ -0,0 +1,41 @@ +{ + "$schema": "https://cloud.uipath.com/draft/2024-12/entry-point", + "$id": "entry-points.json", + "entryPoints": [ + { + "filePath": "main", + "uniqueId": "main", + "type": "function", + "input": { + "type": "object", + "properties": { + "product_id": { + "type": "string" + } + }, + "description": "Input schema.", + "required": [ + "product_id" + ] + }, + "output": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "price": { "type": "number" }, + "category": { "type": "string" }, + "in_stock": { "type": "boolean" }, + "rating": { "type": "number" } + }, + "description": "Output schema.", + "required": [ + "name", + "price", + "category", + "in_stock", + "rating" + ] + } + } + ] +} diff --git a/packages/uipath/samples/list_target_output_key_test/evaluations/eval-sets/default.json b/packages/uipath/samples/list_target_output_key_test/evaluations/eval-sets/default.json new file mode 100644 index 000000000..87c49fdc0 --- /dev/null +++ b/packages/uipath/samples/list_target_output_key_test/evaluations/eval-sets/default.json @@ -0,0 +1,74 @@ +{ + "version": "1.0", + "id": "list-target-output-key-tests", + "name": "List Target Output Key Tests", + "evaluatorRefs": [ + "ListKeysExactMatch", + "ListKeysJsonSimilarity" + ], + "evaluations": [ + { + "id": "headphones-all-match", + "name": "Headphones - all keys match", + "inputs": { + "product_id": "p001" + }, + "evaluationCriterias": { + "ListKeysExactMatch": { + "expectedOutput": { + "name": "Wireless Headphones", + "price": 79.99 + } + }, + "ListKeysJsonSimilarity": { + "expectedOutput": { + "category": "Electronics", + "in_stock": true + } + } + } + }, + { + "id": "shoes-all-match", + "name": "Running Shoes - all keys match", + "inputs": { + "product_id": "p002" + }, + "evaluationCriterias": { + "ListKeysExactMatch": { + "expectedOutput": { + "name": "Running Shoes", + "price": 120.0 + } + }, + "ListKeysJsonSimilarity": { + "expectedOutput": { + "category": "Sports", + "in_stock": false + } + } + } + }, + { + "id": "headphones-wrong-price", + "name": "Headphones - wrong price (should fail)", + "inputs": { + "product_id": "p001" + }, + "evaluationCriterias": { + "ListKeysExactMatch": { + "expectedOutput": { + "name": "Wireless Headphones", + "price": 999.0 + } + }, + "ListKeysJsonSimilarity": { + "expectedOutput": { + "category": "Electronics", + "in_stock": true + } + } + } + } + ] +} diff --git a/packages/uipath/samples/list_target_output_key_test/evaluations/evaluators/list-keys-exact-match.json b/packages/uipath/samples/list_target_output_key_test/evaluations/evaluators/list-keys-exact-match.json new file mode 100644 index 000000000..65e77a8b1 --- /dev/null +++ b/packages/uipath/samples/list_target_output_key_test/evaluations/evaluators/list-keys-exact-match.json @@ -0,0 +1,10 @@ +{ + "version": "1.0", + "id": "ListKeysExactMatch", + "description": "Asserts 'name' and 'price' together using a list of target output keys", + "evaluatorTypeId": "uipath-exact-match", + "evaluatorConfig": { + "name": "ListKeysExactMatch", + "targetOutputKey": ["name", "price"] + } +} diff --git a/packages/uipath/samples/list_target_output_key_test/evaluations/evaluators/list-keys-json-similarity.json b/packages/uipath/samples/list_target_output_key_test/evaluations/evaluators/list-keys-json-similarity.json new file mode 100644 index 000000000..f36f1fdea --- /dev/null +++ b/packages/uipath/samples/list_target_output_key_test/evaluations/evaluators/list-keys-json-similarity.json @@ -0,0 +1,10 @@ +{ + "version": "1.0", + "id": "ListKeysJsonSimilarity", + "description": "Checks 'category' and 'in_stock' together using a list of target output keys", + "evaluatorTypeId": "uipath-json-similarity", + "evaluatorConfig": { + "name": "ListKeysJsonSimilarity", + "targetOutputKey": ["category", "in_stock"] + } +} diff --git a/packages/uipath/samples/list_target_output_key_test/main.py b/packages/uipath/samples/list_target_output_key_test/main.py new file mode 100644 index 000000000..ef7e56c73 --- /dev/null +++ b/packages/uipath/samples/list_target_output_key_test/main.py @@ -0,0 +1,74 @@ +"""Agent demonstrating list targetOutputKey evaluation. + +This agent simulates a product lookup: given a product ID it returns +a structured response with several fields. The evaluators use a list +of keys so that multiple fields can be asserted in a single evaluator +configuration, without comparing the entire output dict. +""" + +from pydantic import BaseModel + +CATALOG: dict[str, dict[str, object]] = { + "p001": { + "name": "Wireless Headphones", + "price": 79.99, + "category": "Electronics", + "in_stock": True, + "rating": 4.5, + }, + "p002": { + "name": "Running Shoes", + "price": 120.0, + "category": "Sports", + "in_stock": False, + "rating": 4.8, + }, + "p003": { + "name": "Coffee Maker", + "price": 49.99, + "category": "Kitchen", + "in_stock": True, + "rating": 4.2, + }, +} + + +class Input(BaseModel): + """Input schema.""" + + product_id: str + + +class Output(BaseModel): + """Output schema.""" + + name: str + price: float + category: str + in_stock: bool + rating: float + + +def main(input_data: Input) -> Output: + """Look up a product by ID and return its details. + + Args: + input_data: Input containing the product ID. + + Returns: + Output with product details. + + Raises: + ValueError: If the product ID is not found. + """ + product = CATALOG.get(input_data.product_id) + if product is None: + raise ValueError(f"Product '{input_data.product_id}' not found") + + return Output( + name=str(product["name"]), + price=float(product["price"]), # type: ignore[arg-type] + category=str(product["category"]), + in_stock=bool(product["in_stock"]), + rating=float(product["rating"]), # type: ignore[arg-type] + ) diff --git a/packages/uipath/samples/list_target_output_key_test/pyproject.toml b/packages/uipath/samples/list_target_output_key_test/pyproject.toml new file mode 100644 index 000000000..ef529b400 --- /dev/null +++ b/packages/uipath/samples/list_target_output_key_test/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "list-target-output-key-test" +version = "0.1.0" +description = "Sample agent demonstrating list targetOutputKey evaluation" +authors = [{ name = "John Doe", email = "john.doe@myemail.com" }] +requires-python = ">=3.11" +dependencies = [ + "uipath" +] + +[tool.uv.sources] +uipath = { path = "../..", editable = true } diff --git a/packages/uipath/samples/list_target_output_key_test/uipath.json b/packages/uipath/samples/list_target_output_key_test/uipath.json new file mode 100644 index 000000000..e2a331e84 --- /dev/null +++ b/packages/uipath/samples/list_target_output_key_test/uipath.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://cloud.uipath.com/draft/2024-12/uipath", + "runtimeOptions": { + "isConversational": false + }, + "packOptions": { + "fileExtensionsIncluded": [], + "filesIncluded": [], + "filesExcluded": [], + "directoriesExcluded": [], + "includeUvLock": true + }, + "functions": { + "main": "main.py:main" + }, + "agents": {} +} diff --git a/packages/uipath/samples/multi-output-agent/evaluations/eval-sets/list-keys.json b/packages/uipath/samples/multi-output-agent/evaluations/eval-sets/list-keys.json new file mode 100644 index 000000000..d1f6e177b --- /dev/null +++ b/packages/uipath/samples/multi-output-agent/evaluations/eval-sets/list-keys.json @@ -0,0 +1,51 @@ +{ + "version": "1.0", + "id": "list-keys-eval-set", + "name": "List Target Output Key Tests", + "evaluatorRefs": [ + "list-keys-exact-match" + ], + "evaluations": [ + { + "id": "list-keys-basic", + "name": "Check multiple keys - order completed", + "inputs": { + "customer_name": "John Doe", + "items": [ + {"name": "Widget", "quantity": 2, "price": 9.99}, + {"name": "Gadget", "quantity": 1, "price": 24.99} + ] + }, + "evaluationCriterias": { + "list-keys-exact-match": { + "expectedOutput": { + "summary": { + "status": "completed", + "total": 44.97 + } + } + } + } + }, + { + "id": "list-keys-mismatch", + "name": "Check multiple keys - wrong total (should fail)", + "inputs": { + "customer_name": "Jane Smith", + "items": [ + {"name": "Book", "quantity": 1, "price": 15.0} + ] + }, + "evaluationCriterias": { + "list-keys-exact-match": { + "expectedOutput": { + "summary": { + "status": "completed", + "total": 999.0 + } + } + } + } + } + ] +} diff --git a/packages/uipath/samples/multi-output-agent/evaluations/evaluators/list-keys-exact-match.json b/packages/uipath/samples/multi-output-agent/evaluations/evaluators/list-keys-exact-match.json new file mode 100644 index 000000000..d2b685bb3 --- /dev/null +++ b/packages/uipath/samples/multi-output-agent/evaluations/evaluators/list-keys-exact-match.json @@ -0,0 +1,10 @@ +{ + "version": "1.0", + "id": "list-keys-exact-match", + "description": "Exact match on multiple output keys at once (summary.status and summary.total)", + "evaluatorTypeId": "uipath-exact-match", + "evaluatorConfig": { + "name": "ListKeysExactMatch", + "targetOutputKey": ["summary.status", "summary.total"] + } +} diff --git a/packages/uipath/samples/runtime-simulations-agent/input.json b/packages/uipath/samples/runtime-simulations-agent/input.json new file mode 100644 index 000000000..9bfb2eef8 --- /dev/null +++ b/packages/uipath/samples/runtime-simulations-agent/input.json @@ -0,0 +1,4 @@ +{ + "code": "def add(a, b):\n return a+b\n\ndef divide(a,b):\n return a/b", + "language": "python" +} diff --git a/packages/uipath/samples/runtime-simulations-agent/main.py b/packages/uipath/samples/runtime-simulations-agent/main.py new file mode 100644 index 000000000..46440b459 --- /dev/null +++ b/packages/uipath/samples/runtime-simulations-agent/main.py @@ -0,0 +1,186 @@ +"""Coding agent that reviews code and suggests improvements. + +This sample demonstrates the --simulation flag: the three tool functions +(check_syntax, check_style, suggest_improvements) are decorated with @mockable, +so they can be intercepted by an LLM during a simulated run instead of +requiring a real linter or compiler to be installed. + +Run with real tools: + uipath run main.py:main -f input.json + +Run with simulation (no real tools needed): + uipath run main.py:main -f input.json --simulation "$(cat simulation.json)" +""" + +import logging + +from pydantic import BaseModel +from pydantic.dataclasses import dataclass + +from uipath.eval.mocks import ExampleCall, mockable +from uipath.tracing import traced + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Input / Output models +# --------------------------------------------------------------------------- + + +@dataclass +class CodeReviewInput: + code: str + language: str = "python" + + +class SyntaxResult(BaseModel): + valid: bool + errors: list[str] = [] + + +class StyleResult(BaseModel): + score: int # 0-100 + violations: list[str] = [] + + +class ImprovementResult(BaseModel): + suggestions: list[str] = [] + refactored_snippet: str = "" + + +class CodeReviewOutput(BaseModel): + syntax: SyntaxResult + style: StyleResult + improvements: ImprovementResult + summary: str + + +# --------------------------------------------------------------------------- +# Mockable tool functions +# --------------------------------------------------------------------------- + +CHECK_SYNTAX_EXAMPLES = [ + ExampleCall( + id="valid-python", + input='{"code": "def hello():\\n return 42", "language": "python"}', + output='{"valid": true, "errors": []}', + ), + ExampleCall( + id="syntax-error", + input='{"code": "def hello(\\n return 42", "language": "python"}', + output='{"valid": false, "errors": ["SyntaxError: unexpected EOF"]}', + ), +] + + +@traced(name="check_syntax", span_type="tool") +@mockable(example_calls=CHECK_SYNTAX_EXAMPLES) +async def check_syntax(code: str, language: str = "python") -> SyntaxResult: + """Check code for syntax errors using the language's parser. + + Args: + code: Source code to check. + language: Programming language (default: python). + + Returns: + SyntaxResult with valid flag and list of error messages. + """ + if language != "python": + return SyntaxResult(valid=True, errors=[]) + + try: + compile(code, "", "exec") + return SyntaxResult(valid=True, errors=[]) + except SyntaxError as exc: + return SyntaxResult(valid=False, errors=[str(exc)]) + + +CHECK_STYLE_EXAMPLES = [ + ExampleCall( + id="clean-code", + input='{"code": "def hello():\\n return 42\\n", "language": "python"}', + output='{"score": 95, "violations": []}', + ), + ExampleCall( + id="style-issues", + input='{"code": "def hello( ):\\n return 42", "language": "python"}', + output='{"score": 60, "violations": ["E211 whitespace before \'(\'", "W291 trailing whitespace"]}', + ), +] + + +@traced(name="check_style", span_type="tool") +@mockable(example_calls=CHECK_STYLE_EXAMPLES) +async def check_style(code: str, language: str = "python") -> StyleResult: + """Run style checks (e.g. PEP 8 for Python) on the provided code. + + Args: + code: Source code to check. + language: Programming language (default: python). + + Returns: + StyleResult with a 0-100 score and list of style violations. + """ + # Real implementation would call ruff / pycodestyle / eslint etc. + # For demo purposes we return a perfect score when not simulated. + return StyleResult(score=100, violations=[]) + + +SUGGEST_IMPROVEMENTS_EXAMPLES = [ + ExampleCall( + id="basic-function", + input='{"code": "def add(a, b):\\n return a + b"}', + output=( + '{"suggestions": ["Add type annotations", "Add a docstring"],' + ' "refactored_snippet": "def add(a: int, b: int) -> int:\\n ' + "'''Return the sum of a and b.'''\\n return a + b\"}" + ), + ) +] + + +@traced(name="suggest_improvements", span_type="tool") +@mockable(example_calls=SUGGEST_IMPROVEMENTS_EXAMPLES) +async def suggest_improvements(code: str) -> ImprovementResult: + """Analyse code and return actionable improvement suggestions. + + Args: + code: Source code to analyse. + + Returns: + ImprovementResult with suggestions and an optional refactored snippet. + """ + # Real implementation would call an LLM or static analysis tool. + return ImprovementResult(suggestions=[], refactored_snippet=code) + + +# --------------------------------------------------------------------------- +# Agent entrypoint +# --------------------------------------------------------------------------- + + +@traced(name="main") +async def main(input: CodeReviewInput) -> CodeReviewOutput: + """Orchestrate three code-review tools and produce a unified report. + + Each tool call creates its own OpenTelemetry span with span_type="tool", + which enables trajectory-based evaluation and simulation. + """ + syntax = await check_syntax(input.code, input.language) + style = await check_style(input.code, input.language) + improvements = await suggest_improvements(input.code) + + issues = len(syntax.errors) + len(style.violations) + summary = ( + f"Found {issues} issue(s). " + f"Style score: {style.score}/100. " + f"{len(improvements.suggestions)} improvement suggestion(s)." + ) + + return CodeReviewOutput( + syntax=syntax, + style=style, + improvements=improvements, + summary=summary, + ) diff --git a/packages/uipath/samples/runtime-simulations-agent/pyproject.toml b/packages/uipath/samples/runtime-simulations-agent/pyproject.toml new file mode 100644 index 000000000..335c55783 --- /dev/null +++ b/packages/uipath/samples/runtime-simulations-agent/pyproject.toml @@ -0,0 +1,14 @@ +[project] +name = "runtime-simulations-agent" +version = "0.0.1" +description = "Code review agent demonstrating runtime simulation" +authors = [{ name = "UiPath", email = "python-sdk@uipath.com" }] +dependencies = [ + "uipath", +] +requires-python = ">=3.11" + +[dependency-groups] +dev = [ + "uipath-dev", +] diff --git a/packages/uipath/samples/runtime-simulations-agent/simulation.json b/packages/uipath/samples/runtime-simulations-agent/simulation.json new file mode 100644 index 000000000..d89bb253f --- /dev/null +++ b/packages/uipath/samples/runtime-simulations-agent/simulation.json @@ -0,0 +1,15 @@ +{ + "enabled": true, + "toolsToSimulate": [ + { + "name": "check_syntax" + }, + { + "name": "check_style" + }, + { + "name": "suggest_improvements" + } + ], + "instructions": "You are simulating a code review system. Given a tool name and its input arguments, produce a realistic JSON response that matches the tool's output schema.\n\n- check_syntax: return {\"valid\": , \"errors\": [, ...]}. If the code looks syntactically correct return valid=true and an empty errors list. Otherwise list the syntax errors.\n- check_style: return {\"score\": <0-100>, \"violations\": [, ...]}. Evaluate PEP 8 compliance for Python code. Deduct points for missing spaces, missing type annotations, etc.\n- suggest_improvements: return {\"suggestions\": [, ...], \"refactored_snippet\": \"\"}. Suggest concrete improvements such as adding type hints, docstrings, or handling edge cases (e.g. division by zero)." +} \ No newline at end of file diff --git a/packages/uipath/samples/runtime-simulations-agent/uipath.json b/packages/uipath/samples/runtime-simulations-agent/uipath.json new file mode 100644 index 000000000..9b02c2654 --- /dev/null +++ b/packages/uipath/samples/runtime-simulations-agent/uipath.json @@ -0,0 +1,5 @@ +{ + "functions": { + "main": "main.py:main" + } +} diff --git a/packages/uipath/samples/simulate-component-agent/input.json b/packages/uipath/samples/simulate-component-agent/input.json new file mode 100644 index 000000000..68093d7d0 --- /dev/null +++ b/packages/uipath/samples/simulate-component-agent/input.json @@ -0,0 +1,4 @@ +{ + "city": "London", + "days": 3 +} diff --git a/packages/uipath/samples/simulate-component-agent/main.py b/packages/uipath/samples/simulate-component-agent/main.py new file mode 100644 index 000000000..499e92c92 --- /dev/null +++ b/packages/uipath/samples/simulate-component-agent/main.py @@ -0,0 +1,121 @@ +"""Weather forecast agent demonstrating per-component simulation. + +This sample shows the new ``components`` simulation format where each tool +has its own simulation strategy and instructions, routed to the +simulate-component API instead of a local LLM. + +Run with real tools (no weather API — returns hardcoded defaults): + uipath run main -f input.json + +Run with per-component simulation (routes each tool call to the API): + uipath run main -f input.json --simulation "$(cat simulation.json)" + +Debug with per-component simulation: + uipath debug main -f input.json --simulation "$(cat simulation.json)" +""" + +from pydantic import BaseModel +from pydantic.dataclasses import dataclass + +from uipath.eval.mocks import mockable +from uipath.tracing import traced + +# --------------------------------------------------------------------------- +# Input / Output models +# --------------------------------------------------------------------------- + + +@dataclass +class WeatherInput: + city: str + days: int = 3 + + +class CurrentWeather(BaseModel): + city: str + temperature: float # Celsius + condition: str + humidity: int # percent + + +class ForecastDay(BaseModel): + date: str # YYYY-MM-DD + high: float + low: float + condition: str + + +class WeatherReport(BaseModel): + current: CurrentWeather + forecast: list[ForecastDay] + summary: str + + +# --------------------------------------------------------------------------- +# Mockable tool functions +# --------------------------------------------------------------------------- + + +@traced(name="get_current_weather", span_type="tool") +@mockable() +async def get_current_weather(city: str) -> CurrentWeather: + """Fetch current weather conditions for a city from an external weather API. + + Args: + city: Name of the city (e.g. "London", "New York"). + + Returns: + CurrentWeather with temperature, condition, and humidity. + """ + # Real implementation would call a weather API such as OpenWeatherMap. + # Returns hardcoded defaults when not simulated. + return CurrentWeather(city=city, temperature=20.0, condition="unknown", humidity=50) + + +@traced(name="get_forecast", span_type="tool") +@mockable() +async def get_forecast(city: str, days: int = 3) -> list[ForecastDay]: + """Retrieve a multi-day weather forecast for a city. + + Args: + city: Name of the city. + days: Number of forecast days to retrieve (default: 3). + + Returns: + List of ForecastDay objects, one per requested day. + """ + # Real implementation would call a forecast API. + # Returns an empty list when not simulated. + return [] + + +# --------------------------------------------------------------------------- +# Agent entry point +# --------------------------------------------------------------------------- + + +@traced(name="main") +async def main(input: WeatherInput) -> WeatherReport: + """Fetch current weather and forecast for a city and produce a report. + + Args: + input: WeatherInput with city name and number of forecast days. + + Returns: + WeatherReport combining current conditions, forecast, and a summary. + """ + current = await get_current_weather(input.city) + forecast = await get_forecast(input.city, input.days) + + issues = [] + if current.humidity > 80: + issues.append("high humidity") + if current.temperature < 0: + issues.append("freezing temperatures") + + alert = f" Alerts: {', '.join(issues)}." if issues else "" + summary = ( + f"{input.city}: {current.temperature}°C, {current.condition}." + f" {len(forecast)}-day forecast available.{alert}" + ) + return WeatherReport(current=current, forecast=forecast, summary=summary) diff --git a/packages/uipath/samples/simulate-component-agent/pyproject.toml b/packages/uipath/samples/simulate-component-agent/pyproject.toml new file mode 100644 index 000000000..c4228e861 --- /dev/null +++ b/packages/uipath/samples/simulate-component-agent/pyproject.toml @@ -0,0 +1,17 @@ +[project] +name = "simulate-component-agent" +version = "0.0.1" +description = "Weather forecast agent demonstrating per-component simulation" +authors = [{ name = "UiPath", email = "python-sdk@uipath.com" }] +dependencies = [ + "uipath", +] +requires-python = ">=3.11" + +[dependency-groups] +dev = [ + "uipath-dev", +] + +[tool.uv.sources] +uipath = { path = "../..", editable = true } diff --git a/packages/uipath/samples/simulate-component-agent/simulation.json b/packages/uipath/samples/simulate-component-agent/simulation.json new file mode 100644 index 000000000..fc87bd527 --- /dev/null +++ b/packages/uipath/samples/simulate-component-agent/simulation.json @@ -0,0 +1,73 @@ +{ + "enabled": true, + "components": [ + { + "componentId": "get_current_weather", + "componentType": "tool", + "simulationStrategy": 0, + "simulationInstruction": "Return realistic current weather for the given city. Use typical seasonal temperatures for the Northern Hemisphere in winter. the humidity should always be 70%", + "outputSchema": { + "type": "object", + "properties": { + "city": { + "type": "string" + }, + "temperature": { + "type": "number", + "description": "Temperature in Celsius" + }, + "condition": { + "type": "string", + "description": "e.g. cloudy, rainy, sunny" + }, + "humidity": { + "type": "integer", + "minimum": 0, + "maximum": 100 + } + }, + "required": [ + "city", + "temperature", + "condition", + "humidity" + ] + } + }, + { + "componentId": "get_forecast", + "componentType": "tool", + "simulationStrategy": 0, + "simulationInstruction": "Return a realistic multi-day weather forecast for the given city.", + "outputSchema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "date": { + "type": "string", + "description": "YYYY-MM-DD" + }, + "high": { + "type": "number", + "description": "High temperature in Celsius" + }, + "low": { + "type": "number", + "description": "Low temperature in Celsius" + }, + "condition": { + "type": "string" + } + }, + "required": [ + "date", + "high", + "low", + "condition" + ] + } + } + } + ] +} \ No newline at end of file diff --git a/packages/uipath/samples/simulate-component-agent/uipath.json b/packages/uipath/samples/simulate-component-agent/uipath.json new file mode 100644 index 000000000..a991b6914 --- /dev/null +++ b/packages/uipath/samples/simulate-component-agent/uipath.json @@ -0,0 +1,6 @@ +{ + "id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "functions": { + "main": "main.py:main" + } +} diff --git a/packages/uipath/specs/uipath.schema.json b/packages/uipath/specs/uipath.schema.json index 8f2a550f8..2ee966e82 100644 --- a/packages/uipath/specs/uipath.schema.json +++ b/packages/uipath/specs/uipath.schema.json @@ -9,6 +9,11 @@ "type": "string", "description": "Reference to this JSON schema for editor support" }, + "id": { + "type": "string", + "format": "uuid", + "description": "Stable unique identifier for the project, minted once on the first 'uipath init' and preserved for its lifetime. Used as the package 'projectId' at pack time. Do not change it." + }, "runtimeOptions": { "type": "object", "description": "Runtime behavior configuration", diff --git a/packages/uipath/specs/uipath.spec.md b/packages/uipath/specs/uipath.spec.md index 5c7599faa..9679bf223 100644 --- a/packages/uipath/specs/uipath.spec.md +++ b/packages/uipath/specs/uipath.spec.md @@ -9,6 +9,7 @@ The `uipath.json` file is a configuration file for UiPath projects that defines ```json { "$schema": "https://cloud.uipath.com/draft/2024-12/uipath", + "id": "00000000-0000-0000-0000-000000000000", "runtimeOptions": { ... }, "designOptions": { ... }, "packOptions": { ... }, @@ -20,7 +21,29 @@ The `uipath.json` file is a configuration file for UiPath projects that defines ## Configuration Sections -### 1. `runtimeOptions` +### 1. `id` + +Stable unique identifier (GUID) for the project, minted once on the first `uipath init` and preserved for its lifetime. Used as the package `projectId` at pack time. + +**Properties:** + +| Property | Type | Required | Default | Description | +|----------|------|----------|---------|-------------| +| `id` | `string` (uuid) | No | minted on first `uipath init` | Stable identifier for the project. Do not change it. | + +> Do not change or remove `id`. It identifies your project consistently wherever it is deployed and run. Changing it makes the project look like a brand-new, unrelated one, so you lose the link to everything previously published and tracked under the old id. `uipath pack` rejects an `id` that is not a valid GUID. + +**Example:** + +```json +{ + "id": "00000000-0000-0000-0000-000000000001" +} +``` + +--- + +### 2. `runtimeOptions` Controls runtime behavior of your UiPath project. @@ -42,7 +65,7 @@ Controls runtime behavior of your UiPath project. --- -### 2. `designOptions` +### 3. `designOptions` Design-time configuration and preferences. @@ -57,7 +80,7 @@ Design-time configuration and preferences. --- -### 3. `packOptions` +### 4. `packOptions` Controls which files and directories are included or excluded when packaging your project. @@ -87,7 +110,7 @@ Controls which files and directories are included or excluded when packaging you --- -### 4. `functions` +### 5. `functions` Defines entrypoints for pure Python scripts. Each key is a friendly name for the entrypoint, and each value specifies the file path and function name. @@ -128,6 +151,7 @@ Defines entrypoints for pure Python scripts. Each key is a friendly name for the ```json { "$schema": "https://cloud.uipath.com/draft/2024-12/uipath", + "id": "00000000-0000-0000-0000-000000000001", "runtimeOptions": { "isConversational": false }, @@ -218,6 +242,11 @@ The complete JSON Schema is available in `uipath.schema.json`: "type": "string", "description": "Reference to this JSON schema for editor support" }, + "id": { + "type": "string", + "format": "uuid", + "description": "Stable unique identifier for the project, minted once on the first 'uipath init' and preserved for its lifetime. Used as the package 'projectId' at pack time. Do not change it." + }, "runtimeOptions": { "type": "object", "description": "Runtime behavior configuration", diff --git a/packages/uipath/src/uipath/_cli/__init__.py b/packages/uipath/src/uipath/_cli/__init__.py index aa6e177e8..f12d46560 100644 --- a/packages/uipath/src/uipath/_cli/__init__.py +++ b/packages/uipath/src/uipath/_cli/__init__.py @@ -7,7 +7,7 @@ from uipath._cli._utils._context import CliContext from uipath._utils._logs import setup_logging -from uipath._utils.constants import DOTENV_FILE +from uipath.platform.constants import DOTENV_FILE # Windows console uses codepages (e.g. cp1252) that can't encode Unicode # characters used by Rich spinners (Braille) and emoji output. @@ -45,6 +45,7 @@ "server": "cli_server", "register": "cli_register", "debug": "cli_debug", + "list-models": "cli_list_models", "assets": "services.cli_assets", "buckets": "services.cli_buckets", "context-grounding": "services.cli_context_grounding", diff --git a/packages/uipath/src/uipath/_cli/_auth/_auth_server.py b/packages/uipath/src/uipath/_cli/_auth/_auth_server.py index 4433b1c20..673d545b3 100644 --- a/packages/uipath/src/uipath/_cli/_auth/_auth_server.py +++ b/packages/uipath/src/uipath/_cli/_auth/_auth_server.py @@ -1,4 +1,5 @@ import asyncio +import hmac import http.server import json import os @@ -10,15 +11,6 @@ PORT = 6234 -# Custom exception for token received -class TokenReceivedSignal(Exception): - """Exception raised when a token is successfully received.""" - - def __init__(self, token_data): - self.token_data = token_data - super().__init__("Token received successfully") - - def make_request_handler_class( state, code_verifier, token_callback, domain, redirect_uri, client_id ): @@ -29,12 +21,60 @@ def log_message(self, format, *args) -> None: # do nothing pass + def _is_host_allowed(self) -> bool: + """Reject requests whose Host header is not loopback. + + Defends against DNS rebinding since the legitimate flow + always lands on localhost. + """ + host = self.headers.get("Host", "") + hostname = host.rsplit(":", 1)[0] + return hostname in ("localhost", "127.0.0.1") + + def _handle_host_error(self) -> bool: + """Return True if a host error was identified and handled (403).""" + if not self._is_host_allowed(): + self.send_error(403, "Invalid host") + return True + return False + + def _state_is_valid(self) -> bool: + """Validate the OAuth state supplied.""" + received = self.headers.get("X-Auth-State", "") + return hmac.compare_digest(received, state) + + def _handle_state_error(self) -> bool: + """Return True if a state error was identified and handled (403).""" + if not self._state_is_valid(): + self.send_error(403, "Invalid or missing state") + return True + return False + + def _read_json_body(self): + """Read and parse the JSON request body. + + Returns the decoded object, or None if the + expected headers are missing or body is malformed. + """ + try: + content_length = int(self.headers["Content-Length"]) + post_data = self.rfile.read(content_length) + return json.loads(post_data.decode("utf-8")) + except (KeyError, TypeError, ValueError): + self.send_error(400, "Invalid request") + return None + def do_POST(self): """Handle POST requests to /set_token.""" + if self._handle_host_error(): + return if self.path == "/set_token": - content_length = int(self.headers["Content-Length"]) - post_data = self.rfile.read(content_length) - token_data = json.loads(post_data.decode("utf-8")) + if self._handle_state_error(): + return + + token_data = self._read_json_body() + if token_data is None: + return self.send_response(200) self.end_headers() @@ -44,9 +84,13 @@ def do_POST(self): token_callback(token_data) elif self.path == "/log": - content_length = int(self.headers["Content-Length"]) - post_data = self.rfile.read(content_length) - logs = json.loads(post_data.decode("utf-8")) + if self._handle_state_error(): + return + + logs = self._read_json_body() + if logs is None: + return + # Write logs to .uipath/.error_log file uipath_dir = os.path.join(os.getcwd(), ".uipath") os.makedirs(uipath_dir, exist_ok=True) @@ -66,6 +110,8 @@ def do_POST(self): def do_GET(self): """Handle GET requests by serving index.html.""" + if self._handle_host_error(): + return # Always serve index.html regardless of the path try: index_path = os.path.join(os.path.dirname(__file__), "index.html") @@ -86,16 +132,6 @@ def do_GET(self): except FileNotFoundError: self.send_error(404, "File not found") - def end_headers(self): - self.send_header("Access-Control-Allow-Origin", "*") - self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") - self.send_header("Access-Control-Allow-Headers", "Content-Type") - super().end_headers() - - def do_OPTIONS(self): - self.send_response(200) - self.end_headers() - return SimpleHTTPSRequestHandler @@ -149,7 +185,7 @@ def create_server(self, state, code_verifier, domain): self.redirect_uri, self.client_id, ) - self.httpd = socketserver.TCPServer(("", self.port), handler) + self.httpd = socketserver.TCPServer(("127.0.0.1", self.port), handler) return self.httpd def _run_server(self): diff --git a/packages/uipath/src/uipath/_cli/_auth/_auth_service.py b/packages/uipath/src/uipath/_cli/_auth/_auth_service.py index c89968683..b594e8fe1 100644 --- a/packages/uipath/src/uipath/_cli/_auth/_auth_service.py +++ b/packages/uipath/src/uipath/_cli/_auth/_auth_service.py @@ -8,7 +8,14 @@ from uipath._cli._auth._utils import get_parsed_token_data from uipath._cli._utils._console import ConsoleLogger from uipath._utils._auth import update_env_file -from uipath.platform.common import ExternalApplicationService, TokenData +from uipath.platform.common import TokenData +from uipath.platform.constants import ( + ENV_BASE_URL, + ENV_ORGANIZATION_ID, + ENV_TENANT_ID, + ENV_UIPATH_ACCESS_TOKEN, +) +from uipath.platform.external_applications import ExternalApplicationService from ._utils import update_auth_file @@ -61,9 +68,9 @@ async def _authenticate_client_credentials(self): ) env_vars = { - "UIPATH_ACCESS_TOKEN": token_data.access_token, - "UIPATH_URL": external_app_service._base_url, - "UIPATH_ORGANIZATION_ID": get_parsed_token_data(token_data).get("prt_id"), + ENV_UIPATH_ACCESS_TOKEN: token_data.access_token, + ENV_BASE_URL: external_app_service._base_url, + ENV_ORGANIZATION_ID: get_parsed_token_data(token_data).get("prt_id"), } if tenant_name: @@ -71,7 +78,7 @@ async def _authenticate_client_credentials(self): auth_session = AuthSession(self._domain) auth_session.update_token_data(token_data) tenant_info = await auth_session.resolve_tenant_info(self._tenant) - env_vars["UIPATH_TENANT_ID"] = tenant_info["tenant_id"] + env_vars[ENV_TENANT_ID] = tenant_info["tenant_id"] else: self._console.warning("Could not extract tenant from --base-url.") update_env_file(env_vars) @@ -90,10 +97,10 @@ async def _authenticate_authorization_code(self) -> None: update_env_file( { - "UIPATH_ACCESS_TOKEN": token_data.access_token, - "UIPATH_URL": uipath_url, - "UIPATH_TENANT_ID": tenant_info["tenant_id"], - "UIPATH_ORGANIZATION_ID": tenant_info["organization_id"], + ENV_UIPATH_ACCESS_TOKEN: token_data.access_token, + ENV_BASE_URL: uipath_url, + ENV_TENANT_ID: tenant_info["tenant_id"], + ENV_ORGANIZATION_ID: tenant_info["organization_id"], } ) @@ -110,9 +117,9 @@ async def _authenticate_authorization_code(self) -> None: async def _can_reuse_existing_token(self, auth_session: AuthSession) -> bool: if ( - os.getenv("UIPATH_URL") - and os.getenv("UIPATH_TENANT_ID") - and os.getenv("UIPATH_ORGANIZATION_ID") + os.getenv(ENV_BASE_URL) + and os.getenv(ENV_TENANT_ID) + and os.getenv(ENV_ORGANIZATION_ID) ): try: await auth_session.ensure_valid_token() diff --git a/packages/uipath/src/uipath/_cli/_auth/_auth_session.py b/packages/uipath/src/uipath/_cli/_auth/_auth_session.py index cd51a583b..3d50b1a17 100644 --- a/packages/uipath/src/uipath/_cli/_auth/_auth_session.py +++ b/packages/uipath/src/uipath/_cli/_auth/_auth_session.py @@ -3,6 +3,7 @@ import click from uipath.platform.common import TokenData +from uipath.platform.constants import ENV_UIPATH_ACCESS_TOKEN from uipath.platform.identity import IdentityService from uipath.platform.portal import ( PortalService as PlatformPortalService, @@ -95,7 +96,7 @@ async def ensure_valid_token(self): def finalize(token_data: TokenData): self.update_token_data(token_data) update_auth_file(token_data) - update_env_file({"UIPATH_ACCESS_TOKEN": token_data.access_token}) + update_env_file({ENV_UIPATH_ACCESS_TOKEN: token_data.access_token}) if exp is not None and float(exp) > time.time(): finalize(auth_data) diff --git a/packages/uipath/src/uipath/_cli/_auth/_url_utils.py b/packages/uipath/src/uipath/_cli/_auth/_url_utils.py index 844968762..6bebe7a23 100644 --- a/packages/uipath/src/uipath/_cli/_auth/_url_utils.py +++ b/packages/uipath/src/uipath/_cli/_auth/_url_utils.py @@ -2,6 +2,8 @@ from typing import Tuple from urllib.parse import urlparse +from uipath.platform.constants import ENV_BASE_URL + from .._utils._console import ConsoleLogger console = ConsoleLogger() @@ -26,7 +28,7 @@ def resolve_domain(base_url: str | None, environment: str | None) -> str: return domain if environment is None: - uipath_url = os.getenv("UIPATH_URL") + uipath_url = os.getenv(ENV_BASE_URL) if uipath_url: parsed = urlparse(uipath_url) if parsed.scheme and parsed.netloc: diff --git a/packages/uipath/src/uipath/_cli/_auth/index.html b/packages/uipath/src/uipath/_cli/_auth/index.html index a361e73de..08f81d81b 100644 --- a/packages/uipath/src/uipath/_cli/_auth/index.html +++ b/packages/uipath/src/uipath/_cli/_auth/index.html @@ -519,6 +519,9 @@

Authenticate CLI

async function sendLogs(logs) { await fetch(`${baseUrl}/log`, { method: 'POST', + headers: { + 'X-Auth-State': "__PY_REPLACE_EXPECTED_STATE__" + }, body: JSON.stringify(logs) }); } @@ -559,6 +562,9 @@

Authenticate CLI

await sendLogs(logs); await fetch(`${baseUrl}/set_token`, { method: 'POST', + headers: { + 'X-Auth-State': state + }, body: JSON.stringify(tokenData) }); diff --git a/packages/uipath/src/uipath/_cli/_chat/_bridge.py b/packages/uipath/src/uipath/_cli/_chat/_bridge.py index 24c1be024..397b7454e 100644 --- a/packages/uipath/src/uipath/_cli/_chat/_bridge.py +++ b/packages/uipath/src/uipath/_cli/_chat/_bridge.py @@ -4,7 +4,7 @@ import json import logging import os -import uuid +from collections import deque from typing import Any from urllib.parse import urlparse @@ -14,18 +14,34 @@ UiPathConversationEvent, UiPathConversationExchangeEndEvent, UiPathConversationExchangeEvent, - UiPathConversationInterruptEndEvent, - UiPathConversationInterruptEvent, + UiPathConversationExecutingToolCallEvent, UiPathConversationMessageEvent, - UiPathConversationToolCallConfirmationInterruptStartEvent, - UiPathConversationToolCallConfirmationValue, + UiPathConversationToolCallConfirmationEvent, + UiPathConversationToolCallEndEvent, + UiPathConversationToolCallEvent, ) from uipath.core.triggers import UiPathResumeTrigger +from uipath.platform.constants import ( + ENV_BASE_URL, + ENV_ORGANIZATION_ID, + ENV_TENANT_ID, + ENV_UIPATH_ACCESS_TOKEN, + HEADER_INTERNAL_ACCOUNT_ID, + HEADER_INTERNAL_TENANT_ID, +) from uipath.runtime.chat import UiPathChatProtocol from uipath.runtime.context import UiPathRuntimeContext logger = logging.getLogger(__name__) +# Type for tool call resume values (confirmToolCall or endToolCall payloads) +ToolResumeValue = ( + UiPathConversationToolCallConfirmationEvent | UiPathConversationToolCallEndEvent +) + +# Wrapper that pairs a resume value with its tool_call_id for keyed matching +ToolResumeItem = dict[str, Any] # {"tool_call_id": str, "value": ToolResumeValue} + class CASErrorId: """Error IDs for the Conversational Agent Service (CAS), matching the Temporal backend.""" @@ -107,6 +123,7 @@ def __init__( exchange_id: str, headers: dict[str, str], auth: dict[str, Any] | None = None, + end_exchange: bool = True, ): """Initialize the WebSocket chat bridge. @@ -116,6 +133,8 @@ def __init__( exchange_id: The exchange ID for this session headers: HTTP headers to send during connection auth: Optional authentication data to send during connection + end_exchange: Whether to send the exchange-end event to CAS on + completion. """ self.websocket_url = websocket_url self.websocket_path = websocket_path @@ -123,12 +142,33 @@ def __init__( self.exchange_id = exchange_id self.auth = auth self.headers = headers + self.end_exchange = end_exchange self._client: Any | None = None self._connected_event = asyncio.Event() - # Interrupt state for HITL round-trip - self._interrupt_end_event = asyncio.Event() - self._interrupt_end_value: UiPathConversationInterruptEndEvent | None = None + # --- Tool call resume state --- + # When the LLM invokes multiple tools in one turn, the client can send + # back confirmToolCall / endToolCall responses concurrently and in any + # order. Three data structures coordinate matching each response to the + # correct wait_for_resume() call: + # + # 1. _expected_tool_call_ids (deque): + # Ordered queue of tool_call_ids populated by emit_interrupt_event() + # (called by the runtime BEFORE each wait_for_resume()). Tells + # wait_for_resume() WHICH tool_call_id it should consume next. + # + # 2. _tool_resume_results (dict): + # Responses that arrived BEFORE wait_for_resume() was called for that + # tool_call_id. When wait_for_resume() runs, it checks here first + # and returns immediately if a match exists — no blocking needed. + # + # 3. _tool_resume_pending (dict of Futures): + # Created by wait_for_resume() when the response hasn't arrived yet. + # When the response later arrives in _handle_conversation_event, the + # Future is resolved and wait_for_resume() unblocks. + self._tool_resume_results: dict[str, ToolResumeItem] = {} + self._tool_resume_pending: dict[str, asyncio.Future[ToolResumeItem]] = {} + self._expected_tool_call_ids: deque[str] = deque() self._current_message_id: str | None = None # Set CAS_WEBSOCKET_DISABLED when using the debugger to prevent websocket errors from @@ -233,6 +273,14 @@ async def disconnect(self) -> None: finally: await self._cleanup_client() + def _require_client(self) -> Any: + client = self._client + if client is None: + raise RuntimeError("WebSocket client not connected. Call connect() first.") + if not self._connected_event.is_set() and not self._websocket_disabled: + raise RuntimeError("WebSocket client not in connected state") + return client + async def emit_message_event( self, message_event: UiPathConversationMessageEvent ) -> None: @@ -244,11 +292,7 @@ async def emit_message_event( Raises: RuntimeError: If client is not connected """ - if self._client is None: - raise RuntimeError("WebSocket client not connected. Call connect() first.") - - if not self._connected_event.is_set() and not self._websocket_disabled: - raise RuntimeError("WebSocket client not in connected state") + client = self._require_client() try: # Wrap message event with conversation/exchange IDs @@ -269,7 +313,7 @@ async def emit_message_event( f"SocketIOChatBridge is in debug mode. Not sending event: {json.dumps(event_data)}" ) else: - await self._client.emit("ConversationEvent", event_data) + await client.emit("ConversationEvent", event_data) # Store the current message ID, used for emitting interrupt events. self._current_message_id = message_event.message_id @@ -278,17 +322,45 @@ async def emit_message_event( logger.error(f"Error sending conversation event to WebSocket: {e}") raise RuntimeError(f"Failed to send conversation event: {e}") from e + async def emit_meta_event(self, meta_event: dict[str, Any]) -> None: + """Send an exchange-scoped conversation metadata event.""" + client = self._require_client() + + try: + event = UiPathConversationEvent( + conversation_id=self.conversation_id, + exchange=UiPathConversationExchangeEvent( + exchange_id=self.exchange_id, + meta_event=meta_event, + ), + ) + event_data = event.model_dump(mode="json", exclude_none=True, by_alias=True) + + if self._websocket_disabled: + logger.info( + "SocketIOChatBridge is in debug mode. Not sending event: %s", + json.dumps(event_data), + ) + else: + await client.emit("ConversationEvent", event_data) + except Exception as e: + logger.error(f"Error sending conversation event to WebSocket: {e}") + raise RuntimeError(f"Failed to send conversation event: {e}") from e + async def emit_exchange_end_event(self) -> None: """Send an exchange end event. + When end_exchange is False the exchange is left open — the event is not + sent to CAS so a downstream consumer can continue and end it later. + Raises: RuntimeError: If client is not connected """ - if self._client is None: - raise RuntimeError("WebSocket client not connected. Call connect() first.") + if not self.end_exchange: + logger.info("end_exchange is False; leaving the exchange open.") + return - if not self._connected_event.is_set() and not self._websocket_disabled: - raise RuntimeError("WebSocket client not in connected state") + client = self._require_client() try: exchange_end_event = UiPathConversationEvent( @@ -308,7 +380,7 @@ async def emit_exchange_end_event(self) -> None: f"SocketIOChatBridge is in debug mode. Not sending event: {json.dumps(event_data)}" ) else: - await self._client.emit("ConversationEvent", event_data) + await client.emit("ConversationEvent", event_data) except Exception as e: logger.error(f"Error sending conversation event to WebSocket: {e}") @@ -324,11 +396,7 @@ async def emit_exchange_error_event(self, error: Exception) -> None: Args: error: The exception that caused the error. """ - if self._client is None: - raise RuntimeError("WebSocket client not connected. Call connect() first.") - - if not self._connected_event.is_set() and not self._websocket_disabled: - raise RuntimeError("WebSocket client not in connected state") + client = self._require_client() # Extract and map error to CAS-specific error ID and message. cas_error_id, cas_message = _resolve_cas_error(error) @@ -356,75 +424,123 @@ async def emit_exchange_error_event(self, error: Exception) -> None: f"SocketIOChatBridge is in debug mode. Not sending event: {json.dumps(event_data)}" ) else: - await self._client.emit("ConversationEvent", event_data) + await client.emit("ConversationEvent", event_data) except Exception as e: logger.error(f"Error sending exchange error event to WebSocket: {e}") raise RuntimeError(f"Failed to send exchange error event: {e}") from e async def emit_interrupt_event(self, resume_trigger: UiPathResumeTrigger): - if self._client and self._connected_event.is_set(): - try: - # Clear previous interrupt state and generate new interrupt_id - self._interrupt_id = str(uuid.uuid4()) - - # Ensure we have a valid message_id - if self._current_message_id is None: - raise RuntimeError( - "Cannot emit interrupt event: no current message_id set" - ) - - # Ensure api_resume is not None - if resume_trigger.api_resume is None: - raise RuntimeError( - "Cannot emit interrupt event: api_resume is None" - ) - - interrupt_event = UiPathConversationEvent( - conversation_id=self.conversation_id, - exchange=UiPathConversationExchangeEvent( - exchange_id=self.exchange_id, - message=UiPathConversationMessageEvent( - message_id=self._current_message_id, - interrupt=UiPathConversationInterruptEvent( - interrupt_id=self._interrupt_id, - start=UiPathConversationToolCallConfirmationInterruptStartEvent( - type="uipath_cas_tool_call_confirmation", - value=UiPathConversationToolCallConfirmationValue( - **resume_trigger.api_resume.request - ), - ), - ), - ), - ), - ) + """Register the trigger's tool_call_id for the upcoming wait_for_resume(). - event_data = interrupt_event.model_dump( - mode="json", exclude_none=True, by_alias=True - ) - if self._websocket_disabled: - logger.info( - f"SocketIOChatBridge is in debug mode. Not sending event: {json.dumps(event_data)}" - ) - else: - await self._client.emit("ConversationEvent", event_data) - except Exception as e: - logger.warning(f"Error sending interrupt event: {e}") + Does not emit any websocket event — tool confirmation and execution + events are handled elsewhere. The runtime calls this immediately + before wait_for_resume() for each trigger, so we record the + tool_call_id here so wait_for_resume() knows which response to match. + """ + if resume_trigger.api_resume and isinstance( + resume_trigger.api_resume.request, dict + ): + tool_call_id = resume_trigger.api_resume.request.get("tool_call_id") + if isinstance(tool_call_id, str) and tool_call_id: + self._expected_tool_call_ids.append(tool_call_id) + + async def emit_executing_tool_call_event( + self, + tool_call_id: str, + tool_input: dict[str, Any] | None = None, + ) -> None: + """Emit an executingToolCall event. + + Called by the runtime loop after a tool-call confirmation resumes + to signal that the tool is about to execute with the final input. + """ + if not self._current_message_id: + return + + executing_event = UiPathConversationMessageEvent( + message_id=self._current_message_id, + tool_call=UiPathConversationToolCallEvent( + tool_call_id=tool_call_id, + executing=UiPathConversationExecutingToolCallEvent( + input=tool_input, + ), + ), + ) + await self.emit_message_event(executing_event) async def wait_for_resume(self) -> dict[str, Any]: - """Wait for the interrupt_end event to be received. + """Wait for a tool resume event (confirmToolCall or endToolCall). + + Pops the next expected tool_call_id (registered by emit_interrupt_event) + and returns the matching response. Two cases: + + 1. Response already arrived (stored in _tool_resume_results) — return + immediately without blocking. + 2. Response hasn't arrived yet — create a Future in _tool_resume_pending, + block until _handle_conversation_event resolves it. Returns: - Resume data from the interrupt end event + The resume data dict, including ``tool_call_id``. """ - self._interrupt_end_event.clear() - self._interrupt_end_value = None + if not self._expected_tool_call_ids: + raise RuntimeError( + "wait_for_resume() called but no tool_call_id was registered " + "by emit_interrupt_event(). This indicates a caller/protocol mismatch." + ) - await self._interrupt_end_event.wait() + expected_id = self._expected_tool_call_ids.popleft() - if self._interrupt_end_value: - return self._interrupt_end_value.model_dump(mode="python", by_alias=False) - return {} + if expected_id in self._tool_resume_results: + # Response arrived before we got here — return it immediately + item = self._tool_resume_results.pop(expected_id) + else: + # Response hasn't arrived yet — wait for it + future: asyncio.Future[ToolResumeItem] = ( + asyncio.get_running_loop().create_future() + ) + self._tool_resume_pending[expected_id] = future + item = await future + + value = item["value"] + result = value.model_dump(mode="python", by_alias=False) + result["tool_call_id"] = item["tool_call_id"] + return result + + def _resolve_or_store_resume( + self, tool_call_id: str, value: ToolResumeValue + ) -> None: + """Route an incoming confirmToolCall/endToolCall to the correct consumer. + + Called from _handle_conversation_event when a tool resume response + arrives from the client. Two cases: + + 1. wait_for_resume() is already waiting (Future in _tool_resume_pending) + — resolve the Future so it unblocks immediately. + 2. wait_for_resume() hasn't been called yet for this tool_call_id + — store in _tool_resume_results so it's found instantly when + wait_for_resume() runs later. + """ + item: ToolResumeItem = {"tool_call_id": tool_call_id, "value": value} + if tool_call_id in self._tool_resume_pending: + future = self._tool_resume_pending.pop(tool_call_id) + if not future.done(): + future.set_result(item) + else: + # Future was cancelled or already resolved — store the payload + # so a subsequent wait_for_resume() can still find it. + logger.warning( + f"Resume for tool_call_id={tool_call_id} — " + "future already done, storing as fallback." + ) + self._tool_resume_results[tool_call_id] = item + else: + if tool_call_id in self._tool_resume_results: + logger.warning( + f"Duplicate resume for tool_call_id={tool_call_id} — " + "overwriting previously stored result." + ) + self._tool_resume_results[tool_call_id] = item @property def is_connected(self) -> bool: @@ -458,17 +574,12 @@ async def _handle_conversation_event( if ( parsed_event.exchange and parsed_event.exchange.message - and parsed_event.exchange.message.interrupt - and parsed_event.exchange.message.interrupt.end + and (tool_call := parsed_event.exchange.message.tool_call) ): - interrupt = parsed_event.exchange.message.interrupt - - if interrupt.interrupt_id == self._interrupt_id: - logger.info( - f"Received endInterrupt for interrupt_id: {self._interrupt_id}" - ) - self._interrupt_end_value = interrupt.end - self._interrupt_end_event.set() + if confirm := tool_call.confirm: + self._resolve_or_store_resume(tool_call.tool_call_id, confirm) + elif end := tool_call.end: + self._resolve_or_store_resume(tool_call.tool_call_id, end) except Exception as e: logger.warning(f"Error parsing conversation event: {e}") @@ -506,7 +617,7 @@ def get_chat_bridge( assert context.exchange_id is not None, "exchange_id must be set in context" # Extract host from UIPATH_URL - base_url = os.environ.get("UIPATH_URL") + base_url = os.environ.get(ENV_BASE_URL) if not base_url: raise RuntimeError( "UIPATH_URL environment variable required for conversational mode" @@ -531,20 +642,30 @@ def get_chat_bridge( # Build headers from context headers = { - "Authorization": f"Bearer {os.environ.get('UIPATH_ACCESS_TOKEN', '')}", - "X-UiPath-Internal-TenantId": f"{context.tenant_id}" - or os.environ.get("UIPATH_TENANT_ID", ""), - "X-UiPath-Internal-AccountId": f"{context.org_id}" - or os.environ.get("UIPATH_ORGANIZATION_ID", ""), + "Authorization": f"Bearer {os.environ.get(ENV_UIPATH_ACCESS_TOKEN, '')}", + HEADER_INTERNAL_TENANT_ID: context.tenant_id + or os.environ.get(ENV_TENANT_ID, ""), + HEADER_INTERNAL_ACCOUNT_ID: context.org_id + or os.environ.get(ENV_ORGANIZATION_ID, ""), "X-UiPath-ConversationId": context.conversation_id, } + # Conversation owner id (conversationalService.conversationalUserId) that CAS forwards via + # FpsProperties; always sent when present. It's there for RunAsMe=false, where the unattended + # robot's token subject is the robot account rather than the conversation owner, so CAS validates + # this presented id against conversation.user_id on the handshake instead of the token subject. + # Sent as a header (not a query param) to keep it out of access / load-balancer logs. + conversational_user_id = getattr(context, "conversational_user_id", None) + if conversational_user_id: + headers["X-UiPath-Internal-ConversationalUserId"] = conversational_user_id + return SocketIOChatBridge( websocket_url=websocket_url, websocket_path=websocket_path, conversation_id=context.conversation_id, exchange_id=context.exchange_id, headers=headers, + end_exchange=getattr(context, "end_exchange", True), ) diff --git a/packages/uipath/src/uipath/_cli/_chat/_voice_bridge.py b/packages/uipath/src/uipath/_cli/_chat/_voice_bridge.py new file mode 100644 index 000000000..4240f28c0 --- /dev/null +++ b/packages/uipath/src/uipath/_cli/_chat/_voice_bridge.py @@ -0,0 +1,292 @@ +"""Voice tool-call session — persistent socket.io connection to CAS.""" + +from __future__ import annotations + +import asyncio +import logging +import os +from collections.abc import Awaitable, Callable +from copy import deepcopy +from enum import Enum +from typing import Any +from urllib.parse import urlparse + +from pydantic import ValidationError + +from uipath.core.chat import ( + UiPathVoiceToolCallMessage, + UiPathVoiceToolCallRequest, + UiPathVoiceToolCallResult, +) +from uipath.platform.constants import ( + ENV_BASE_URL, + ENV_ORGANIZATION_ID, + ENV_TENANT_ID, + ENV_UIPATH_ACCESS_TOKEN, + HEADER_INTERNAL_ACCOUNT_ID, + HEADER_INTERNAL_TENANT_ID, +) +from uipath.runtime.context import UiPathRuntimeContext + +logger = logging.getLogger(__name__) + + +_ATTEMPT_CAS_SOCKET_CONNECTION_TIMEOUT_SECONDS = 15.0 +_INFLIGHT_TOOL_DRAIN_AFTER_AGENT_END_TIMEOUT_SECONDS = 30.0 + + +class VoiceToolCallSessionError(RuntimeError): + pass + + +class VoiceSessionEndReason(str, Enum): + COMPLETED = "completed" + DISCONNECTED = "disconnected" + READY_EMIT_FAILED = "ready_emit_failed" + + +class VoiceEvent(str, Enum): + """CAS voice-session protocol events (excludes socket.io lifecycle).""" + + TOOL_CALL = "voice_tool_call" # received + SESSION_ENDED = "voice_session_ended" # received + TOOLS_READY = "voice_tools_ready" # sent + TOOL_RESULT = "voice_tool_result" # sent + + +ToolHandler = Callable[ + [UiPathVoiceToolCallRequest], Awaitable[UiPathVoiceToolCallResult] +] + + +class VoiceToolCallSession: + """Socket.io session with CAS for tool-call traffic. + + Receives `voice_tool_call` batches, emits one `voice_tool_result` per + `callId`, exits on `voice_session_ended` or disconnect. CAS pulls + agent config from Orchestrator directly; this session carries only + tool calls. + """ + + def __init__( + self, + url: str, + socketio_path: str, + headers: dict[str, str], + tool_handler: ToolHandler, + ) -> None: + self._url = url + self._socketio_path = socketio_path + self._headers = headers + self._tool_handler = tool_handler + self._client: Any = None + self._done = asyncio.Event() + self._in_flight: set[asyncio.Task[None]] = set() + self._end_reason: VoiceSessionEndReason | None = None + self._end_detail: dict[str, Any] = {} + + @property + def end_detail(self) -> dict[str, Any]: + """CAS payload from voice_session_ended, preserved for the job runtime.""" + return deepcopy(self._end_detail) + + async def run(self) -> VoiceSessionEndReason: + """Connect, dispatch tool calls until session ends, then disconnect. + + Raises: + VoiceToolCallSessionError: If connecting to CAS fails. + """ + from socketio import AsyncClient # type: ignore[import-untyped] + + self._client = AsyncClient(logger=False, engineio_logger=False) + self._client.on("connect", self._handle_connect) + self._client.on("disconnect", self._handle_disconnect) + self._client.on(VoiceEvent.TOOL_CALL, self._handle_tool_call) + self._client.on(VoiceEvent.SESSION_ENDED, self._handle_session_ended) + + try: + await asyncio.wait_for( + self._client.connect( + url=self._url, + socketio_path=self._socketio_path, + headers=self._headers, + transports=["websocket"], + ), + timeout=_ATTEMPT_CAS_SOCKET_CONNECTION_TIMEOUT_SECONDS, + ) + except Exception as exc: + await self._safe_disconnect("after connect-failure") + raise VoiceToolCallSessionError( + f"Failed to connect to CAS voice endpoint: {exc}" + ) from exc + + try: + await self._done.wait() + await self._drain_in_flight() + finally: + await self._safe_disconnect("on shutdown") + + return self._end_reason or VoiceSessionEndReason.DISCONNECTED + + async def _safe_disconnect(self, when: str) -> None: + try: + await self._client.disconnect() + except Exception as exc: + logger.debug("[Voice] disconnect %s raised: %s", when, exc) + + def _end_session(self, reason: VoiceSessionEndReason) -> None: + # First writer wins: a late disconnect must not overwrite COMPLETED. + if self._end_reason is None: + self._end_reason = reason + self._done.set() + + async def _drain_in_flight(self) -> None: + """Wait for in-flight tool tasks to finish, capped by the drain timeout.""" + if not self._in_flight: + return + logger.info( + "[Voice] Session ended with %d in-flight tool task(s); draining (max %.0fs)", + len(self._in_flight), + _INFLIGHT_TOOL_DRAIN_AFTER_AGENT_END_TIMEOUT_SECONDS, + ) + try: + await asyncio.wait_for( + asyncio.gather(*self._in_flight, return_exceptions=True), + timeout=_INFLIGHT_TOOL_DRAIN_AFTER_AGENT_END_TIMEOUT_SECONDS, + ) + except asyncio.TimeoutError: + unfinished = sum(1 for t in self._in_flight if not t.done()) + logger.warning( + "[Voice] %d tool task(s) did not complete within %.0fs of session end", + unfinished, + _INFLIGHT_TOOL_DRAIN_AFTER_AGENT_END_TIMEOUT_SECONDS, + ) + + async def _handle_connect(self) -> None: + logger.info("[Voice] Socket.io connected to CAS") + try: + await self._client.emit(VoiceEvent.TOOLS_READY, {}) + except Exception as exc: + # CAS gates tool dispatch on this event; without it the session is dead. + logger.warning("[Voice] emit voice_tools_ready failed: %s", exc) + self._end_session(VoiceSessionEndReason.READY_EMIT_FAILED) + + async def _handle_disconnect(self) -> None: + logger.info("[Voice] Socket.io disconnected from CAS") + self._end_session(VoiceSessionEndReason.DISCONNECTED) + + async def _handle_tool_call(self, data: dict[str, Any], *_: Any) -> None: + """Spawn a task per call and return — the reader must stay free for `voice_session_ended`.""" + if self._done.is_set(): + return + + try: + message = UiPathVoiceToolCallMessage.model_validate(data) + except ValidationError as exc: + logger.warning("[Voice] invalid voice_tool_call payload: %s", exc) + return + + for call in message.calls: + task = asyncio.create_task(self._execute_tool_call(call)) + self._in_flight.add(task) + task.add_done_callback(self._in_flight.discard) + + async def _execute_tool_call(self, call: UiPathVoiceToolCallRequest) -> None: + """Run one tool call and emit its `voice_tool_result`.""" + logger.info( + "[Voice] voice_tool_call dispatched: %s (%s) args=%s", + call.tool_name, + call.call_id, + call.args, + ) + try: + tool_result = await self._tool_handler(call) + except Exception as exc: + logger.exception("[Voice] Tool call execution failed: %s", call.tool_name) + tool_result = UiPathVoiceToolCallResult(result=str(exc), is_error=True) + + try: + await self._client.emit( + VoiceEvent.TOOL_RESULT, + {"callId": call.call_id, **tool_result.model_dump(by_alias=True)}, + ) + except Exception as exc: + logger.debug( + "[Voice] emit voice_tool_result failed for %s: %s", call.call_id, exc + ) + return + logger.info( + "[Voice] voice_tool_result sent: %s (isError=%s)", + call.call_id, + tool_result.is_error, + ) + + async def _handle_session_ended(self, data: Any = None, *_: Any) -> None: + if self._done.is_set(): + return + + detail = deepcopy(data) if isinstance(data, dict) else {} + self._end_detail = detail + logger.info( + "[Voice] voice_session_ended received " + "(endedBy=%s, callEnded=%s, reason=%s)", + detail.get("endedBy"), + detail.get("callEnded"), + detail.get("reason"), + ) + self._end_session(VoiceSessionEndReason.COMPLETED) + + +def get_voice_bridge( + context: UiPathRuntimeContext, + tool_handler: ToolHandler, +) -> VoiceToolCallSession: + """Factory for a CAS voice tool-call session. + + Raises: + RuntimeError: If UIPATH_URL is not set or invalid. + """ + assert context.conversation_id is not None, "conversation_id must be set in context" + + if cas_host := os.environ.get("CAS_WEBSOCKET_HOST"): + url = f"ws://{cas_host}?conversationId={context.conversation_id}" + socketio_path = "/socket.io" + logger.warning( + f"CAS_WEBSOCKET_HOST is set. Using websocket_url '{url}{socketio_path}'." + ) + else: + base_url = os.environ.get(ENV_BASE_URL) + if not base_url: + raise RuntimeError( + "UIPATH_URL environment variable required for conversational mode" + ) + parsed = urlparse(base_url) + if not parsed.netloc: + raise RuntimeError(f"Invalid UIPATH_URL format: {base_url}") + url = f"wss://{parsed.netloc}?conversationId={context.conversation_id}" + socketio_path = "autopilotforeveryone_/websocket_/socket.io" + + headers = { + "Authorization": f"Bearer {os.environ.get(ENV_UIPATH_ACCESS_TOKEN, '')}", + HEADER_INTERNAL_TENANT_ID: context.tenant_id + or os.environ.get(ENV_TENANT_ID, ""), + HEADER_INTERNAL_ACCOUNT_ID: context.org_id + or os.environ.get(ENV_ORGANIZATION_ID, ""), + "X-UiPath-ConversationId": context.conversation_id, + } + + # Conversation owner id (conversationalService.conversationalUserId) that CAS forwards via + # FpsProperties; always sent when present. It's there for RunAsMe=false, where the unattended + # robot's token subject is the robot account rather than the conversation owner, so CAS validates + # this presented id against conversation.user_id on the handshake instead of the token subject. + # Sent as a header (not a query param) to keep it out of access / load-balancer logs. + conversational_user_id = getattr(context, "conversational_user_id", None) + if conversational_user_id: + headers["X-UiPath-Internal-ConversationalUserId"] = conversational_user_id + + return VoiceToolCallSession( + url=url, + socketio_path=socketio_path, + headers=headers, + tool_handler=tool_handler, + ) diff --git a/packages/uipath/src/uipath/_cli/_debug/_bridge.py b/packages/uipath/src/uipath/_cli/_debug/_bridge.py index 9607398a0..0094bc012 100644 --- a/packages/uipath/src/uipath/_cli/_debug/_bridge.py +++ b/packages/uipath/src/uipath/_cli/_debug/_bridge.py @@ -13,15 +13,28 @@ from uipath.core.serialization import serialize_object from uipath.core.triggers import UiPathResumeTriggerType +from uipath.platform.constants import ( + ENV_BASE_URL, + ENV_UIPATH_ACCESS_TOKEN, + HEADER_FOLDER_KEY, + HEADER_INTERNAL_ACCOUNT_ID, + HEADER_INTERNAL_TENANT_ID, +) from uipath.runtime import ( UiPathBreakpointResult, UiPathRuntimeContext, UiPathRuntimeResult, UiPathRuntimeStatus, ) -from uipath.runtime.debug import UiPathDebugProtocol, UiPathDebugQuitError +from uipath.runtime.debug import ( + DetachedDebugBridge, + UiPathDebugProtocol, + UiPathDebugQuitError, +) from uipath.runtime.events import UiPathRuntimeStateEvent, UiPathRuntimeStatePhase +DebugAttachMode = Literal["signalr", "console", "none"] + logger = logging.getLogger(__name__) @@ -741,6 +754,9 @@ async def _handle_start(self, args: list[Any]) -> None: f"Debug started: breakpoints={self.state.breakpoints}, step_mode={step_mode}" ) + # handle race conditions, runtime connected to debug bridge before the receiver + await self.emit_execution_started() + async def _handle_resume(self, args: list[Any]) -> None: """Handle Resume command from SignalR server. @@ -851,7 +867,7 @@ async def _handle_error(self, error: Any) -> None: def get_remote_debug_bridge(context: UiPathRuntimeContext) -> UiPathDebugProtocol: """Factory to get SignalR debug bridge for remote debugging.""" - uipath_url = os.environ.get("UIPATH_URL") + uipath_url = os.environ.get(ENV_BASE_URL) if not uipath_url or not context.job_id: raise ValueError( "UIPATH_URL and UIPATH_JOB_KEY are required for remote debugging" @@ -861,28 +877,37 @@ def get_remote_debug_bridge(context: UiPathRuntimeContext) -> UiPathDebugProtoco return SignalRDebugBridge( hub_url=signalr_url, - access_token=os.environ.get("UIPATH_ACCESS_TOKEN"), + access_token=os.environ.get(ENV_UIPATH_ACCESS_TOKEN), headers={ - "X-UiPath-Internal-TenantId": context.tenant_id or "", - "X-UiPath-Internal-AccountId": context.org_id or "", - "X-UiPath-FolderKey": context.folder_key or "", + HEADER_INTERNAL_TENANT_ID: context.tenant_id or "", + HEADER_INTERNAL_ACCOUNT_ID: context.org_id or "", + HEADER_FOLDER_KEY: context.folder_key or "", }, ) def get_debug_bridge( - context: UiPathRuntimeContext, verbose: bool = True + context: UiPathRuntimeContext, + verbose: bool = True, + attach: DebugAttachMode | None = None, ) -> UiPathDebugProtocol: """Factory to get appropriate debug bridge based on context. Args: context: The runtime context containing debug configuration. verbose: If True, console bridge shows all state updates. If False, only breakpoints. + attach: Explicit attach mode. When None, falls back to + ``context.job_id``-based selection. Returns: An instance of UiPathDebugBridge suitable for the context. """ - if context.job_id: + if attach == "none": + return DetachedDebugBridge() + if attach == "signalr": return get_remote_debug_bridge(context) - else: + if attach == "console": return ConsoleDebugBridge(verbose=verbose) + if context.job_id: + return get_remote_debug_bridge(context) + return ConsoleDebugBridge(verbose=verbose) diff --git a/packages/uipath/src/uipath/_cli/_errors.py b/packages/uipath/src/uipath/_cli/_errors.py new file mode 100644 index 000000000..feb7006a4 --- /dev/null +++ b/packages/uipath/src/uipath/_cli/_errors.py @@ -0,0 +1,20 @@ +class EntrypointDiscoveryException(Exception): + """Raised when entrypoint auto-discovery fails.""" + + def __init__(self, entrypoints: list[str]): + self.entrypoints = entrypoints + + def get_usage_help(self) -> list[str]: + if self.entrypoints: + lines = ["Available entrypoints:"] + for name in self.entrypoints: + lines.append(f" - {name}") + return lines + return [ + "No entrypoints found.", + "", + "To configure entrypoints, use one of the following:", + " 1. Functions project (uipath.json)", + " 2. Framework-specific project (e.g. langgraph.json, llamaindex.json, openai_agents.json)", + " 3. MCP project (mcp.json)", + ] diff --git a/packages/uipath/src/uipath/_cli/_evals/_progress_reporter.py b/packages/uipath/src/uipath/_cli/_evals/_progress_reporter.py index 7c5114516..b1ac65a65 100644 --- a/packages/uipath/src/uipath/_cli/_evals/_progress_reporter.py +++ b/packages/uipath/src/uipath/_cli/_evals/_progress_reporter.py @@ -16,11 +16,6 @@ from uipath._cli._utils._console import ConsoleLogger from uipath._utils import Endpoint, RequestSpec -from uipath._utils.constants import ( - ENV_EVAL_BACKEND_URL, - ENV_TENANT_ID, - HEADER_INTERNAL_TENANT_ID, -) from uipath.core.events import EventBus from uipath.eval.evaluators import ( BaseEvaluator, @@ -38,6 +33,14 @@ ) from uipath.platform import UiPath from uipath.platform.common import UiPathConfig +from uipath.platform.constants import ( + ENV_EVAL_BACKEND_URL, + ENV_TENANT_ID, + ENV_UIPATH_AGENT_ID, + ENV_UIPATH_PROJECT_FILES_SOURCE, + ENV_UIPATH_PROJECT_ID, + HEADER_INTERNAL_TENANT_ID, +) logger = logging.getLogger(__name__) @@ -101,12 +104,19 @@ def __init__(self): self._client = uipath.api_client self._console = console_logger self._rich_console = Console() - self._project_id = os.getenv("UIPATH_PROJECT_ID", None) - if not self._project_id: + self._project_id = os.getenv(ENV_UIPATH_PROJECT_ID, None) + self._agent_id = os.getenv(ENV_UIPATH_AGENT_ID) or self._project_id + if not self._agent_id: logger.warning( "Cannot report data to StudioWeb. Please set UIPATH_PROJECT_ID." ) + # Map UIPATH_PROJECT_FILES_SOURCE (Local/Cloud) to the backend's + # ProjectFilesSource enum integer. Without this every row the worker + # creates lands as Cloud, and the UI's `?projectFilesSource=1` filter + # never matches local-workspace runs. + self._project_files_source = self._resolve_project_files_source() + self.eval_set_ids: dict[str, str] = {} # Track eval_set_id per execution self.eval_set_run_ids: dict[str, str] = {} self.evaluators: dict[str, Any] = {} @@ -351,7 +361,7 @@ def _extract_usage_from_spans( """Extract token usage and cost from OpenTelemetry spans. Args: - spans: List of ReadableSpan objects from agent execution + spans: List of ReadableSpan objects from workload execution Returns: Dictionary with tokens, completionTokens, promptTokens, and cost @@ -1089,6 +1099,29 @@ def _collect_coded_results( evaluator_runs.append(evaluator_run) return evaluator_runs, evaluator_scores_list + @staticmethod + def _resolve_project_files_source() -> int | None: + raw = os.getenv(ENV_UIPATH_PROJECT_FILES_SOURCE) + if not raw: + return None + normalized = raw.strip().lower() + if normalized == "local": + return 1 + if normalized == "cloud": + return 0 + try: + return int(normalized) + except ValueError: + logger.warning( + f"Unrecognized UIPATH_PROJECT_FILES_SOURCE value: {raw!r}; ignoring." + ) + return None + + def _project_files_source_field(self) -> dict[str, int]: + if self._project_files_source is None: + return {} + return {"projectFilesSource": self._project_files_source} + def _update_eval_run_spec( self, assertion_runs: list[dict[str, Any]], @@ -1115,6 +1148,7 @@ def _update_eval_run_spec( }, "completionMetrics": {"duration": int(execution_time * 1000)}, "assertionRuns": assertion_runs, + **self._project_files_source_field(), } # Legacy backend expects payload wrapped in "request" field @@ -1133,7 +1167,7 @@ def _update_eval_run_spec( return RequestSpec( method="PUT", endpoint=Endpoint( - f"{self._get_endpoint_prefix()}execution/agents/{self._project_id}/{endpoint_suffix}evalRun" + f"{self._get_endpoint_prefix()}execution/agents/{self._agent_id}/{endpoint_suffix}evalRun" ), json=payload, headers=self._tenant_header(), @@ -1166,6 +1200,7 @@ def _update_coded_eval_run_spec( }, "completionMetrics": {"duration": int(execution_time * 1000)}, "evaluatorRuns": evaluator_runs, + **self._project_files_source_field(), } # Log the payload for debugging coded eval run updates @@ -1181,7 +1216,7 @@ def _update_coded_eval_run_spec( return RequestSpec( method="PUT", endpoint=Endpoint( - f"{self._get_endpoint_prefix()}execution/agents/{self._project_id}/{endpoint_suffix}evalRun" + f"{self._get_endpoint_prefix()}execution/agents/{self._agent_id}/{endpoint_suffix}evalRun" ), json=payload, headers=self._tenant_header(), @@ -1235,6 +1270,7 @@ def _create_eval_run_spec( "evalSnapshot": eval_snapshot, # Backend expects integer status "status": EvaluationStatus.IN_PROGRESS.value, + **self._project_files_source_field(), } # Legacy backend expects payload wrapped in "request" field @@ -1253,7 +1289,7 @@ def _create_eval_run_spec( return RequestSpec( method="POST", endpoint=Endpoint( - f"{self._get_endpoint_prefix()}execution/agents/{self._project_id}/{endpoint_suffix}evalRun" + f"{self._get_endpoint_prefix()}execution/agents/{self._agent_id}/{endpoint_suffix}evalRun" ), json=payload, headers=self._tenant_header(), @@ -1283,7 +1319,7 @@ def _create_eval_set_run_spec( eval_set_id_value = str(uuid.uuid5(uuid.NAMESPACE_DNS, eval_set_id)) inner_payload: dict[str, Any] = { - "agentId": self._project_id, + "agentId": self._agent_id, "evalSetId": eval_set_id_value, "agentSnapshot": agent_snapshot.model_dump(by_alias=True), # Backend expects integer status @@ -1291,6 +1327,7 @@ def _create_eval_set_run_spec( "numberOfEvalsExecuted": no_of_evals, # Source is required by the backend (0 = coded SDK) "source": 0, + **self._project_files_source_field(), } # Both coded and legacy send payload directly at root level @@ -1309,7 +1346,7 @@ def _create_eval_set_run_spec( return RequestSpec( method="POST", endpoint=Endpoint( - f"{self._get_endpoint_prefix()}execution/agents/{self._project_id}/{endpoint_suffix}evalSetRun" + f"{self._get_endpoint_prefix()}execution/agents/{self._agent_id}/{endpoint_suffix}evalSetRun" ), json=payload, headers=self._tenant_header(), @@ -1353,6 +1390,7 @@ def _update_eval_set_run_spec( # Backend expects integer status "status": status.value, "evaluatorScores": evaluator_scores_list, + **self._project_files_source_field(), } # Legacy backend expects payload wrapped in "request" field @@ -1374,7 +1412,7 @@ def _update_eval_set_run_spec( return RequestSpec( method="PUT", endpoint=Endpoint( - f"{self._get_endpoint_prefix()}execution/agents/{self._project_id}/{endpoint_suffix}evalSetRun" + f"{self._get_endpoint_prefix()}execution/agents/{self._agent_id}/{endpoint_suffix}evalSetRun" ), json=payload, headers=self._tenant_header(), @@ -1406,12 +1444,12 @@ def _get_eval_runs_spec( if is_coded: endpoint_path = ( - f"{prefix}execution/agents/{self._project_id}/coded/" + f"{prefix}execution/agents/{self._agent_id}/coded/" f"evalSets/{eval_set_id}/evalSetRuns/{eval_set_run_id}/evalRuns" ) else: endpoint_path = ( - f"{prefix}execution/agents/{self._project_id}/" + f"{prefix}execution/agents/{self._agent_id}/" f"evalSets/{eval_set_id}/evalSetRuns/{eval_set_run_id}/evalRuns" ) @@ -1420,10 +1458,14 @@ def _get_eval_runs_spec( f"eval_set_run_id={eval_set_run_id}, evaluation_id={evaluation_id}, coded={is_coded}" ) + # The backend's listing endpoint filters by projectFilesSource + + # cloudUserId so the UI only shows the caller's local rows. Mirror + # that here so resume lookups match the row written by the same + # worker session. return RequestSpec( method="GET", endpoint=Endpoint(endpoint_path), - params={}, # No query params needed - evalSetRunId is in the path + params=self._project_files_source_field(), headers=self._tenant_header(), ) diff --git a/packages/uipath/src/uipath/_cli/_evals/_telemetry.py b/packages/uipath/src/uipath/_cli/_evals/_telemetry.py index ad9549a6c..64843972b 100644 --- a/packages/uipath/src/uipath/_cli/_evals/_telemetry.py +++ b/packages/uipath/src/uipath/_cli/_evals/_telemetry.py @@ -19,6 +19,8 @@ EvaluationEvents, ) from uipath.platform.common import UiPathConfig +from uipath.platform.constants import ENV_TENANT_ID +from uipath.runtime.errors import UiPathBaseRuntimeError from uipath.telemetry._track import is_telemetry_enabled, track_event logger = logging.getLogger(__name__) @@ -56,17 +58,23 @@ def __init__(self) -> None: self._eval_run_info: dict[str, dict[str, Any]] = {} self._current_eval_set_run_id: str | None = None self._current_entrypoint: str | None = None + self._current_agent_type: str | None = None @staticmethod - def _get_agent_type(entrypoint: str) -> str: - """Determine agent type from entrypoint. - - Args: - entrypoint: The entrypoint path. - - Returns: - "LowCode" if entrypoint is "agent.json", "Coded" otherwise. + def _resolve_agent_type(agent_type: str | None, entrypoint: str | None) -> str: + """Emit the historical ``"LowCode"`` / ``"Coded"`` wire values. + + Application Insights dashboards filter on those two exact strings. + When the factory supplies a modern label (e.g. ``"uipath_lowcode"``, + ``"uipath_coded"``) we normalize; when it supplies nothing we fall + back to the pre-refactor entrypoint check (``agent.json`` ⇒ + low-code) so no in-flight consumer breaks. """ + if agent_type is not None: + if "lowcode" in agent_type.lower() or "low_code" in agent_type.lower(): + return "LowCode" + return "Coded" + # Fallback: pre-refactor entrypoint-based derivation. if entrypoint == "agent.json": return "LowCode" return "Coded" @@ -104,6 +112,7 @@ async def _on_eval_set_run_created(self, event: EvalSetRunCreatedEvent) -> None: "eval_set_id": event.eval_set_id, "eval_set_run_id": eval_set_run_id, "entrypoint": event.entrypoint, + "agent_type": event.agent_type, "no_of_evals": event.no_of_evals, "evaluator_count": len(event.evaluators), } @@ -111,6 +120,7 @@ async def _on_eval_set_run_created(self, event: EvalSetRunCreatedEvent) -> None: # Store for child events self._current_eval_set_run_id = eval_set_run_id self._current_entrypoint = event.entrypoint + self._current_agent_type = event.agent_type properties: dict[str, Any] = { "EvalSetId": event.eval_set_id, @@ -118,7 +128,9 @@ async def _on_eval_set_run_created(self, event: EvalSetRunCreatedEvent) -> None: "Entrypoint": event.entrypoint, "EvalCount": event.no_of_evals, "EvaluatorCount": len(event.evaluators), - "AgentType": self._get_agent_type(event.entrypoint), + "AgentType": self._resolve_agent_type( + event.agent_type, event.entrypoint + ), "Runtime": "URT", } @@ -155,7 +167,9 @@ async def _on_eval_run_created(self, event: EvalRunCreatedEvent) -> None: # Add entrypoint and agent type if self._current_entrypoint: properties["Entrypoint"] = self._current_entrypoint - properties["AgentType"] = self._get_agent_type(self._current_entrypoint) + properties["AgentType"] = self._resolve_agent_type( + self._current_agent_type, self._current_entrypoint + ) self._enrich_properties(properties) @@ -206,7 +220,9 @@ async def _on_eval_run_updated(self, event: EvalRunUpdatedEvent) -> None: if self._current_entrypoint: properties["Entrypoint"] = self._current_entrypoint - properties["AgentType"] = self._get_agent_type(self._current_entrypoint) + properties["AgentType"] = self._resolve_agent_type( + self._current_agent_type, self._current_entrypoint + ) if trace_id: properties["TraceId"] = trace_id @@ -223,15 +239,15 @@ async def _on_eval_run_updated(self, event: EvalRunUpdatedEvent) -> None: ) if event.exception_details: - properties["ErrorType"] = type( - event.exception_details.exception - ).__name__ - properties["ErrorMessage"] = str(event.exception_details.exception)[ - :500 - ] + exception = event.exception_details.exception + properties["ErrorType"] = type(exception).__name__ + properties["ErrorMessage"] = str(exception)[:500] properties["IsRuntimeException"] = ( event.exception_details.runtime_exception ) + if isinstance(exception, UiPathBaseRuntimeError): + properties["ErrorCode"] = exception.error_info.code + properties["ErrorCategory"] = exception.error_info.category.value self._enrich_properties(properties) @@ -270,7 +286,9 @@ async def _on_eval_set_run_updated(self, event: EvalSetRunUpdatedEvent) -> None: if set_info.get("entrypoint"): properties["Entrypoint"] = set_info["entrypoint"] - properties["AgentType"] = self._get_agent_type(set_info["entrypoint"]) + properties["AgentType"] = self._resolve_agent_type( + set_info.get("agent_type"), set_info["entrypoint"] + ) properties["Runtime"] = "URT" @@ -298,6 +316,7 @@ async def _on_eval_set_run_updated(self, event: EvalSetRunUpdatedEvent) -> None: self._current_eval_set_run_id = None self._current_entrypoint = None + self._current_agent_type = None except Exception as e: logger.debug(f"Error tracking eval set run updated: {e}") @@ -308,28 +327,38 @@ def _enrich_properties(self, properties: dict[str, Any]) -> None: Args: properties: The properties dictionary to enrich. """ - # Add UiPath context + from uipath.platform.common._span_utils import resolve_project_id + if UiPathConfig.project_id: properties["ProjectId"] = UiPathConfig.project_id - properties["AgentId"] = UiPathConfig.project_id + if agent_id := resolve_project_id(): + properties["AgentId"] = agent_id - # Get organization ID from UiPathConfig if UiPathConfig.organization_id: properties["CloudOrganizationId"] = UiPathConfig.organization_id - # Get CloudUserId from JWT token - try: - cloud_user_id = get_claim_from_token("sub") - if cloud_user_id: - properties["CloudUserId"] = cloud_user_id - except Exception: - pass # CloudUserId is optional - - # Get tenant ID from environment - tenant_id = os.getenv("UIPATH_TENANT_ID") + cloud_user_id = UiPathConfig.cloud_user_id + if not cloud_user_id: + try: + cloud_user_id = get_claim_from_token("sub") + except Exception: + cloud_user_id = None + if cloud_user_id: + properties["CloudUserId"] = cloud_user_id + + tenant_id = os.getenv(ENV_TENANT_ID) if tenant_id: properties["TenantId"] = tenant_id - # Add source identifier + # Origin of the eval-set run as classified by the caller (e.g. Manual, + # Protegi, FirstSuccessfulRun). The Agents backend forwards the value + # via UIPATH_EVAL_RUN_SOURCE so adoption dashboards can exclude + # auto-triggered runs (e.g. first-successful-run) from user-driven counts. + # Distinct from the `Source` dimension below, which categorises the SDK + # emitter ("uipath-python-cli"), not the run origin. + run_source = os.getenv("UIPATH_EVAL_RUN_SOURCE") + if run_source: + properties["RunSource"] = run_source + properties["Source"] = "uipath-python-cli" properties["ApplicationName"] = "UiPath.Eval" diff --git a/packages/uipath/src/uipath/_cli/_governance/__init__.py b/packages/uipath/src/uipath/_cli/_governance/__init__.py new file mode 100644 index 000000000..fc49b92a1 --- /dev/null +++ b/packages/uipath/src/uipath/_cli/_governance/__init__.py @@ -0,0 +1,16 @@ +"""CLI-side governance helpers. + +Host-only glue that turns provider responses into inputs the runtime +consumes. Owns the YAML → :class:`PolicyIndex` compiler (the runtime +layer stays format-agnostic and only accepts a compiled index). + +Public helpers: + +- :func:`build_policy_index_from_yaml` — parse a YAML policy pack (as + returned by :meth:`GovernancePolicyProvider.get_policy_async`) into + a :class:`uipath.runtime.governance.native.PolicyIndex`. +""" + +from .yaml_index import build_policy_index_from_yaml + +__all__ = ["build_policy_index_from_yaml"] diff --git a/packages/uipath/src/uipath/_cli/_governance/yaml_index.py b/packages/uipath/src/uipath/_cli/_governance/yaml_index.py new file mode 100644 index 000000000..4da02a276 --- /dev/null +++ b/packages/uipath/src/uipath/_cli/_governance/yaml_index.py @@ -0,0 +1,532 @@ +"""YAML → :class:`PolicyIndex` compiler. + +Lives CLI-side so the runtime layer never has to depend on ``pyyaml`` +or know about the wire policy format — the runtime consumes compiled +indexes only. +""" + +from __future__ import annotations + +import logging +from collections.abc import Callable +from typing import Any + +import yaml + +from uipath.core.governance.models import Action, LifecycleHook +from uipath.runtime.governance.native.models import ( + Check, + Condition, + Logic, + PolicyIndex, + PolicyPack, + Rule, + Severity, +) + +logger = logging.getLogger(__name__) + + +_HOOK_MAP: dict[str, LifecycleHook] = { + "before_agent": LifecycleHook.BEFORE_AGENT, + "after_agent": LifecycleHook.AFTER_AGENT, + "before_model": LifecycleHook.BEFORE_MODEL, + "after_model": LifecycleHook.AFTER_MODEL, + "wrap_tool_call": LifecycleHook.TOOL_CALL, + "tool_call": LifecycleHook.TOOL_CALL, + "after_tool": LifecycleHook.AFTER_TOOL, +} + +_ACTION_MAP: dict[str, Action] = { + "block": Action.DENY, + "deny": Action.DENY, + "log": Action.AUDIT, + "audit": Action.AUDIT, + "allow": Action.ALLOW, + "require_approval": Action.ESCALATE, + "escalate": Action.ESCALATE, +} + +_SEVERITY_MAP: dict[str, Severity] = { + "low": Severity.LOW, + "medium": Severity.MEDIUM, + "high": Severity.HIGH, + "critical": Severity.CRITICAL, +} + + +def build_policy_index_from_yaml(yaml_text: str) -> PolicyIndex: + """Parse YAML policy packs into a :class:`PolicyIndex`. + + Unknown check types and malformed rules are skipped with a debug log + (partial packs preferred over failing the whole load); malformed + YAML at the document level raises :class:`yaml.YAMLError`. + """ + index = PolicyIndex() + documents = list(yaml.safe_load_all(yaml_text)) + + for doc in documents: + if not isinstance(doc, dict): + continue + pack = _build_pack(doc) + if pack is not None and pack.rules: + index.add_pack(pack) + + logger.debug( + "Built PolicyIndex from YAML: packs=%s, rules=%d", + index.pack_names, + index.total_rules, + ) + return index + + +def _build_pack(data: dict[str, Any]) -> PolicyPack | None: + """Build a PolicyPack from one YAML document.""" + name = data.get("standard") or data.get("name") + if not name: + logger.warning("Skipping pack: missing 'standard'/'name' field") + return None + + default_action_str = data.get("default_action", "block") + default_action = _ACTION_MAP.get(default_action_str, Action.DENY) + + rules: list[Rule] = [] + for i, rule_data in enumerate(data.get("rules", []) or []): + if not isinstance(rule_data, dict): + continue + rule = _build_rule(rule_data, default_action, i) + if rule is not None: + rules.append(rule) + + return PolicyPack( + name=str(name), + version=str(data.get("version", "1.0.0")), + description=str(data.get("description", "")), + rules=rules, + ) + + +def _build_rule( + data: dict[str, Any], default_action: Action, index: int +) -> Rule | None: + """Build a single Rule from a YAML rule entry.""" + hook = _HOOK_MAP.get(data.get("hook", "before_model")) + if hook is None: + logger.warning( + "Skipping rule %s: unknown hook %r", data.get("id"), data.get("hook") + ) + return None + + action_str = data.get("action") + action = ( + _ACTION_MAP.get(action_str, default_action) if action_str else default_action + ) + + default_sev = "high" if action == Action.DENY else "medium" + severity = _SEVERITY_MAP.get(data.get("severity", default_sev), Severity.HIGH) + + checks = _build_checks( + data.get("checks", []) or [], + action, + mapped_to_uipath=bool(data.get("mapped_to_uipath", False)), + policy_enabled=bool(data.get("policy_enabled", True)), + ) + + # If checks were declared but none could be parsed (e.g. all unknown + # types), skip the rule. A rule with zero checks "always matches" in + # the evaluator, so keeping it would make it fire on every request. + declared = data.get("checks", []) or [] + if declared and not checks: + logger.warning( + "Skipping rule %s: none of its %d declared check(s) could be parsed", + data.get("id"), + len(declared), + ) + return None + + return Rule( + rule_id=str(data.get("id", f"RULE-{index}")), + name=str(data.get("name", data.get("id", f"RULE-{index}"))), + clause=str(data.get("clause", data.get("owasp_ref", ""))), + hook=hook, + action=action, + severity=severity, + checks=checks, + enabled=bool(data.get("enabled", True)), + description=str(data.get("description", "")), + ) + + +def _build_checks( + checks_data: list[dict[str, Any]], + default_action: Action, + *, + mapped_to_uipath: bool = False, + policy_enabled: bool = True, +) -> list[Check]: + """Build the checks list for a rule. + + ``mapped_to_uipath`` / ``policy_enabled`` are rule-level flags read + by ``guardrail_fallback`` checks so the per-check condition can + decide whether to fire the compensating governance call. + """ + checks: list[Check] = [] + for check_data in checks_data: + if not isinstance(check_data, dict): + continue + check = _build_check( + check_data, + default_action, + mapped_to_uipath=mapped_to_uipath, + policy_enabled=policy_enabled, + ) + if check is not None: + checks.append(check) + return checks + + +# --------------------------------------------------------------------------- +# Per-check-type condition builders +# +# Each returns ``(conditions, default_message)`` given the YAML entry for +# one check. The main :func:`_build_check` picks the right builder from +# :data:`_CHECK_BUILDERS` and layers action / logic / message resolution +# on top — keeping the dispatch flat instead of one giant if/elif chain. +# --------------------------------------------------------------------------- + + +def _build_regex_conditions(data: dict[str, Any]) -> tuple[list[Condition], str]: + scope = data.get("scope", ["human", "ai"]) + field = _field_for_scope(scope) + conditions = [ + Condition(operator="regex", field=field, value=pattern) + for pattern in (data.get("patterns", []) or []) + ] + return conditions, f"Pattern matched in {scope}" + + +def _build_budget_conditions(data: dict[str, Any]) -> tuple[list[Condition], str]: + return ( + _gt_conditions_from_keys( + data, + ( + ("max_tool_calls_per_session", "session_state.tool_calls"), + ("max_tool_calls_per_minute", "session_state.tool_calls_per_minute"), + ( + "max_consecutive_tool_calls", + "session_state.consecutive_tool_calls", + ), + ), + ), + "Tool budget exceeded", + ) + + +def _build_tool_allowlist_conditions( + data: dict[str, Any], +) -> tuple[list[Condition], str]: + blocked_tools = data.get("blocked_tools", []) or [] + conditions = ( + [Condition(operator="in_list", field="tool_name", value=blocked_tools)] + if blocked_tools + else [] + ) + return conditions, "Tool not allowed" + + +def _build_parameter_validation_conditions( + data: dict[str, Any], +) -> tuple[list[Condition], str]: + conditions = [ + Condition(operator="regex", field="tool_args", value=pattern) + for pattern in (data.get("additional_patterns", []) or []) + ] + return conditions, "Suspicious pattern in tool parameters" + + +def _build_rate_limit_conditions( + data: dict[str, Any], +) -> tuple[list[Condition], str]: + return ( + _gt_conditions_from_keys( + data, + ( + ("max_llm_calls_per_session", "session_state.llm_calls"), + ("max_llm_calls_per_minute", "session_state.llm_calls_per_minute"), + ), + ), + "Rate limit exceeded", + ) + + +def _build_field_regex_conditions( + data: dict[str, Any], +) -> tuple[list[Condition], str]: + conditions = _make_conditions(data.get("conditions", []) or []) + return conditions, str(data.get("message", "Field regex check failed")) + + +def _build_data_quality_score_conditions( + data: dict[str, Any], +) -> tuple[list[Condition], str]: + field = data.get("field", "tool_result") + conditions: list[Condition] = [] + if data.get("check_encoding", True): + conditions.append( + Condition( + operator="encoding_concern", + field=field, + value={ + "min_confidence": float(data.get("min_confidence", 0.5)), + "max_replacement_ratio": float( + data.get("max_replacement_ratio", 0.05) + ), + "min_corruption_events": int(data.get("min_corruption_events", 2)), + }, + ) + ) + if data.get("check_entropy", True): + conditions.append( + Condition( + operator="entropy_concern", + field=field, + value={ + "min": float(data.get("entropy_min", 1.5)), + "max": float(data.get("entropy_max", 7.5)), + }, + ) + ) + return conditions, str(data.get("message", "")) + + +def _build_incident_taxonomy_conditions( + data: dict[str, Any], +) -> tuple[list[Condition], str]: + field = data.get("field", "model_output") + categories = data.get("categories") + value: dict[str, Any] = {} + if categories: + value["categories"] = list(categories) + conditions = [Condition(operator="incident_concern", field=field, value=value)] + return conditions, str(data.get("message", "")) + + +def _build_commitment_extractor_conditions( + data: dict[str, Any], +) -> tuple[list[Condition], str]: + field = data.get("field", "model_output") + conditions = [ + Condition( + operator="commitment_concern", + field=field, + value={ + "require_amount": bool(data.get("require_amount", True)), + "require_deadline": bool(data.get("require_deadline", False)), + }, + ) + ] + return conditions, str(data.get("message", "")) + + +def _build_sentiment_concern_conditions( + data: dict[str, Any], +) -> tuple[list[Condition], str]: + field = data.get("field", "model_input") + threshold = float(data.get("threshold", -0.3)) + conditions = [ + Condition( + operator="vader_concern", + field=field, + value={"threshold": threshold}, + ) + ] + default_msg = f"Negative sentiment detected (VADER compound <= {threshold})" + return conditions, default_msg + + +def _gt_conditions_from_keys( + data: dict[str, Any], + keys_to_fields: tuple[tuple[str, str], ...], +) -> list[Condition]: + """Emit ``gt`` conditions for each YAML key present in ``data``. + + Shared by budget/rate_limit builders — they only differ in which + (YAML key → CheckContext field) pairs they scan. + """ + return [ + Condition(operator="gt", field=field, value=data[key]) + for key, field in keys_to_fields + if key in data + ] + + +# check_type → builder. ``guardrail_fallback`` is handled inline in +# :func:`_build_check` because it needs the rule-level flags. +_CHECK_BUILDERS: dict[str, Callable[[dict[str, Any]], tuple[list[Condition], str]]] = { + "regex": _build_regex_conditions, + "budget": _build_budget_conditions, + "tool_allowlist": _build_tool_allowlist_conditions, + "parameter_validation": _build_parameter_validation_conditions, + "rate_limit": _build_rate_limit_conditions, + "field_regex": _build_field_regex_conditions, + "data_quality_score": _build_data_quality_score_conditions, + "incident_taxonomy": _build_incident_taxonomy_conditions, + "commitment_extractor": _build_commitment_extractor_conditions, + "sentiment_concern": _build_sentiment_concern_conditions, +} + + +def _build_guardrail_fallback_conditions( + data: dict[str, Any], + *, + mapped_to_uipath: bool, + policy_enabled: bool, +) -> tuple[list[Condition], str]: + """Compensating-control condition. Depends on rule-level flags. + + ``validator`` names which guardrail check the compensating call + should run. The runtime's ``guardrail_fallback`` operator fires + only when the guardrail is mapped to UiPath but disabled. + """ + conditions = [ + Condition( + operator="guardrail_fallback", + field="", + value={ + "validator": str(data.get("validator", "")), + "mapped_to_uipath": mapped_to_uipath, + "policy_enabled": policy_enabled, + }, + ) + ] + default_msg = "Guardrail disabled — compensating check needed." + return conditions, default_msg + + +def _resolve_action(data: dict[str, Any], default_action: Action) -> Action: + """Resolve the check's action against ``_ACTION_MAP`` with a default fallback.""" + action_str = data.get("action") + if not action_str: + return default_action + return _ACTION_MAP.get(action_str, default_action) + + +def _resolve_logic( + data: dict[str, Any], + *, + has_explicit_conditions: bool, + check_type: str, + n_conditions: int, +) -> Logic: + """Resolve the check's ``logic`` field with the right default. + + Multi-pattern shorthand (``regex`` / ``parameter_validation`` + expanded from several patterns for one concept) defaults to ``any`` + — any pattern hitting is a match. An explicit ``conditions:`` list + defaults to ``all`` (all must hold) and must NOT inherit the + pattern-shorthand OR even though ``check_type`` falls back to + ``"regex"``. Explicit ``logic`` in the YAML always wins. + """ + if ( + not has_explicit_conditions + and check_type in ("parameter_validation", "regex") + and n_conditions > 1 + ): + default_logic = "any" + else: + default_logic = "all" + logic_str = str(data.get("logic", default_logic)).lower() + try: + return Logic(logic_str) + except ValueError: + return Logic.ALL + + +def _has_explicit_conditions(raw_conditions: Any) -> bool: + """A ``conditions:`` list is explicit when it holds dicts with ``operator:``.""" + return ( + isinstance(raw_conditions, list) + and bool(raw_conditions) + and isinstance(raw_conditions[0], dict) + and "operator" in raw_conditions[0] + ) + + +def _build_check( + data: dict[str, Any], + default_action: Action, + *, + mapped_to_uipath: bool = False, + policy_enabled: bool = True, +) -> Check | None: + """Build one Check from a YAML check entry. + + Delegates per-check-type condition-building to the small helpers + above (dispatched via :data:`_CHECK_BUILDERS`); the ``guardrail_fallback`` + branch is inline because it needs the rule-level + ``mapped_to_uipath`` / ``policy_enabled`` flags threaded in from + :func:`_build_rule`. Unknown check types are skipped. + """ + raw_conditions = data.get("conditions") + has_explicit_conditions = _has_explicit_conditions(raw_conditions) + check_type = data.get("type", "regex") + + if has_explicit_conditions: + assert isinstance(raw_conditions, list) # narrowed by _has_explicit_conditions + conditions = list(_make_conditions(raw_conditions)) + message = str(data.get("message", "")) + elif check_type == "guardrail_fallback": + conditions, message = _build_guardrail_fallback_conditions( + data, + mapped_to_uipath=mapped_to_uipath, + policy_enabled=policy_enabled, + ) + else: + builder = _CHECK_BUILDERS.get(check_type) + if builder is None: + logger.debug("Skipping check: unknown type %r", check_type) + return None + conditions, message = builder(data) + + if not conditions: + return None + + action = _resolve_action(data, default_action) + message = str(data.get("message", message)) + logic = _resolve_logic( + data, + has_explicit_conditions=has_explicit_conditions, + check_type=check_type, + n_conditions=len(conditions), + ) + return Check(conditions=conditions, action=action, message=message, logic=logic) + + +def _make_conditions(raw: list[dict[str, Any]]) -> list[Condition]: + """Translate a list of YAML condition dicts into Condition objects.""" + out: list[Condition] = [] + for cond in raw: + if not isinstance(cond, dict): + continue + out.append( + Condition( + operator=str(cond.get("operator", "regex")), + field=str(cond.get("field", "model_input")), + value=cond.get("value", ""), + negate=bool(cond.get("negate", False)), + ) + ) + return out + + +def _field_for_scope(scope: list[str] | str) -> str: + """Map a YAML `scope` value to the CheckContext field it targets.""" + if isinstance(scope, str): + scope = [scope] + if "system" in scope or "human" in scope: + return "model_input" + if "ai" in scope: + return "model_output" + if "tool_result" in scope: + return "tool_result" + return "model_input" diff --git a/packages/uipath/src/uipath/_cli/_governance_bootstrap.py b/packages/uipath/src/uipath/_cli/_governance_bootstrap.py new file mode 100644 index 000000000..675b42721 --- /dev/null +++ b/packages/uipath/src/uipath/_cli/_governance_bootstrap.py @@ -0,0 +1,174 @@ +"""Shared host-side governance bootstrap for ``uipath run`` / ``uipath debug``. + +Framework and agent-type labels are forwarded from +:class:`UiPathRuntimeFactorySettings` — each factory advertises its +own; the CLI never classifies the runtime. +""" + +from __future__ import annotations + +import atexit +import logging +from collections.abc import Callable +from dataclasses import dataclass + +from uipath.core.governance import EnforcementMode, PolicyContext +from uipath.core.governance.config import is_governance_enabled +from uipath.platform import UiPath +from uipath.platform.governance import UiPathPlatformGovernanceProvider +from uipath.platform.governance._live_track_event_dispatcher import ( + LiveTrackEventDispatcher, +) +from uipath.runtime import UiPathRuntimeProtocol +from uipath.runtime.governance._audit.base import AuditManager +from uipath.runtime.governance._audit.metadata import GovernanceRuntimeMetadata +from uipath.runtime.governance.native import GovernanceEvaluator +from uipath.runtime.governance.native.guardrail_compensation import ( + GuardrailCompensator, +) +from uipath.runtime.governance.native.models import PolicyIndex +from uipath.runtime.governance.runtime import UiPathGovernedRuntime + +from ._governance import build_policy_index_from_yaml +from ._utils._console import ConsoleLogger + +console = ConsoleLogger() +logger = logging.getLogger(__name__) + +__all__ = [ + "GovernanceBootstrap", + "resolve_governance", +] + + +@dataclass(frozen=True, slots=True) +class GovernanceBootstrap: + """Governance wiring for one CLI run. + + ``dispose`` is idempotent, never raises, and drains the track-event + dispatcher; call it from a ``finally``. An :mod:`atexit` fallback + covers the case where the caller misses it. + """ + + evaluator: GovernanceEvaluator + policy_index: PolicyIndex + enforcement_mode: EnforcementMode + dispose: Callable[[], None] + + def wrap_runtime( + self, + delegate: UiPathRuntimeProtocol, + *, + agent_name: str, + runtime_id: str, + ) -> UiPathGovernedRuntime: + """Wrap a delegate runtime with governance evaluation.""" + return UiPathGovernedRuntime( + delegate, + policy_index=self.policy_index, + enforcement_mode=self.enforcement_mode, + evaluator=self.evaluator, + agent_name=agent_name, + runtime_id=runtime_id, + ) + + +async def resolve_governance( + *, + agent_framework: str | None, + agent_type: str | None, + is_conversational: bool, +) -> GovernanceBootstrap | None: + """Fetch policy + build the governance stack, or ``None`` when disabled. + + ``agent_framework`` and ``agent_type`` are forwarded from + :class:`UiPathRuntimeFactorySettings` and stamped on every audit + event; ``None`` becomes ``"unknown"``. + + ``is_conversational`` is derived by the caller from runtime context + (``bool(ctx.conversation_id)``): ``True`` for a run inside a CAS + conversation, ``False`` otherwise. The value is forwarded verbatim + to :class:`PolicyContext` so the backend can select the + conversational or autonomous policy view. + """ + if not is_governance_enabled(): + return None + + context = PolicyContext(is_conversational=is_conversational) + + try: + sdk = UiPath() + provider = UiPathPlatformGovernanceProvider(service=sdk.governance) + response = await provider.get_policy_async(context) + except Exception as exc: + console.warning( + f"Governance policy fetch failed - continuing without governance: {exc}" + ) + return None + + if response.mode is None or response.mode == EnforcementMode.DISABLED: + return None + if not response.policies: + return None + + try: + policy_index = build_policy_index_from_yaml(response.policies) + except Exception as exc: + console.warning( + f"Governance policy compilation failed - continuing without governance: {exc}" + ) + return None + + # The dispatcher below owns a background thread + atexit hook, so + # every failure path from here on must run ``dispose``. + track_event_dispatcher: LiveTrackEventDispatcher | None = None + + def dispose() -> None: + # Called from CLI ``finally`` — must never raise. + dispatcher = track_event_dispatcher + if dispatcher is None: + return + try: + atexit.unregister(dispatcher.shutdown) + except Exception: + logger.debug("atexit.unregister failed", exc_info=True) + try: + dispatcher.shutdown() + except Exception: + logger.debug("dispatcher shutdown failed", exc_info=True) + + try: + track_event_dispatcher = LiveTrackEventDispatcher(provider) + atexit.register(track_event_dispatcher.shutdown) + + compensator = GuardrailCompensator(provider) + audit_manager = AuditManager( + track_event=track_event_dispatcher.dispatch, + runtime_metadata=GovernanceRuntimeMetadata( + agent_type=agent_type or "unknown", + agent_framework=agent_framework or "unknown", + ), + ) + evaluator = GovernanceEvaluator( + policy_index, + enforcement_mode=response.mode, + audit_manager=audit_manager, + compensator=compensator, + ) + console.info( + f"Governance enabled (mode={response.mode.value}, " + f"packs={list(policy_index.pack_names)})" + ) + except Exception as exc: + dispose() + console.warning( + f"Governance setup failed - continuing without governance: {exc}" + ) + return None + + return GovernanceBootstrap( + evaluator=evaluator, + policy_index=policy_index, + enforcement_mode=response.mode, + dispose=dispose, + ) diff --git a/packages/uipath/src/uipath/_cli/_push/_resolvers.py b/packages/uipath/src/uipath/_cli/_push/_resolvers.py new file mode 100644 index 000000000..e609014d5 --- /dev/null +++ b/packages/uipath/src/uipath/_cli/_push/_resolvers.py @@ -0,0 +1,177 @@ +from typing import AsyncIterator + +from uipath.platform.connections import ConnectionsService +from uipath.platform.errors import EnrichedException, FolderNotFoundException +from uipath.platform.resource_catalog import ( + Resource, + ResourceCatalogService, + ResourceType, +) + +from .._utils._studio_project import ( + ReferencedResourceFolder, + ReferencedResourceRequest, + VirtualResourceRequest, +) +from ..models.runtime_schema import BindingResource, Bindings +from ._resource_actions import CreateReference, CreateVirtual, ResourceAction, Skip + +_NOT_FOUND_SUFFIX = "was not found and will not be added to the solution." + + +async def resolve_bindings( + bindings: Bindings, + resource_catalog: ResourceCatalogService, + connections: ConnectionsService, + supported_virtual_kinds: set[str], +) -> AsyncIterator[ResourceAction]: + """Yield one ResourceAction per importable binding. + + Bindings that should be silently ignored (e.g. guardrail bindings without a + folderPath) are filtered out here. + """ + for binding in bindings.resources: + action = await _resolve_binding( + binding, resource_catalog, connections, supported_virtual_kinds + ) + if action is not None: + yield action + + +async def _resolve_binding( + binding: BindingResource, + resource_catalog: ResourceCatalogService, + connections: ConnectionsService, + supported_virtual_kinds: set[str], +) -> ResourceAction | None: + if binding.resource == "connection": + return await _resolve_connection(binding, resource_catalog, connections) + return await _resolve_regular(binding, resource_catalog, supported_virtual_kinds) + + +async def _resolve_connection( + binding: BindingResource, + resource_catalog: ResourceCatalogService, + connections: ConnectionsService, +) -> ResourceAction | None: + connection_id_value = binding.value.get("ConnectionId") + if connection_id_value is None: + raise ValueError( + f"Connection binding {binding.key!r} is missing required field 'ConnectionId'" + ) + connection_key = connection_id_value.default_value + + try: + connection = await connections.retrieve_async(connection_key) + except EnrichedException: + connector_name = (binding.metadata or {}).get("Connector") + return Skip( + message=( + f"Connection with key '{connection_key}' of type " + f"'{connector_name}' {_NOT_FOUND_SUFFIX}" + ) + ) + + resource_name: str = connection.name + folder_path: str = connection.folder.get("path") + + found = await _find_in_resource_catalog( + resource_catalog, "connection", resource_name, folder_path + ) + if found is None: + return Skip( + message=( + f"Resource '{resource_name}' of type 'connection' at folder path " + f"'{folder_path}' {_NOT_FOUND_SUFFIX}" + ) + ) + return _build_create_reference(found, resource_name) + + +async def _resolve_regular( + binding: BindingResource, + resource_catalog: ResourceCatalogService, + supported_virtual_kinds: set[str], +) -> ResourceAction | None: + name_value = binding.value.get("name") + folder_path_value = binding.value.get("folderPath") + if not folder_path_value: + # guardrail resource, nothing to import + return None + if name_value is None: + raise ValueError(f"Binding {binding.key!r} is missing required field 'name'") + resource_name: str = name_value.default_value + folder_path: str = folder_path_value.default_value + resource_type: str = binding.resource + + found = await _find_in_resource_catalog( + resource_catalog, resource_type, resource_name, folder_path + ) + if found is not None: + return _build_create_reference(found, resource_name) + + if resource_type not in supported_virtual_kinds: + return Skip( + message=( + f"Cannot create virtual resource '{resource_name}' — " + f"kind '{resource_type}' is not supported." + ) + ) + + sub_type: str | None = (binding.metadata or {}).get("SubType") + return CreateVirtual( + request=VirtualResourceRequest( + kind=resource_type, + name=resource_name, + type=sub_type, + ) + ) + + +async def _find_in_resource_catalog( + resource_catalog: ResourceCatalogService, + resource_type: str, + name: str, + folder_path: str, +) -> Resource | None: + """Look up a single resource in the Resource Catalog. + + Returns the first match or None if the catalog can't search this kind, the + folder is unknown, or no resource matches. + """ + catalog_type = next( + (m for m in ResourceType if m.value == resource_type.lower()), None + ) + if catalog_type is None: + return None + + resources = resource_catalog.list_by_type_async( + resource_type=catalog_type, name=name, folder_path=folder_path + ) + try: + return await anext(resources, None) + except FolderNotFoundException: + return None + finally: + await resources.aclose() + + +def _build_create_reference( + found_resource: Resource, resource_name: str +) -> CreateReference: + folder = next(iter(found_resource.folders)) + return CreateReference( + request=ReferencedResourceRequest( + key=found_resource.resource_key, + kind=found_resource.resource_type, + type=found_resource.resource_sub_type, + folder=ReferencedResourceFolder( + folder_key=folder.key, + fully_qualified_name=folder.fully_qualified_name, + path=folder.path, + ), + ), + resource_name=resource_name, + kind=found_resource.resource_type, + sub_type=found_resource.resource_sub_type, + ) diff --git a/packages/uipath/src/uipath/_cli/_push/_resource_actions.py b/packages/uipath/src/uipath/_cli/_push/_resource_actions.py new file mode 100644 index 000000000..4eb76bd57 --- /dev/null +++ b/packages/uipath/src/uipath/_cli/_push/_resource_actions.py @@ -0,0 +1,27 @@ +from dataclasses import dataclass + +from .._utils._studio_project import ( + ReferencedResourceRequest, + VirtualResourceRequest, +) + + +@dataclass(frozen=True, slots=True) +class CreateReference: + request: ReferencedResourceRequest + resource_name: str + kind: str + sub_type: str | None + + +@dataclass(frozen=True, slots=True) +class CreateVirtual: + request: VirtualResourceRequest + + +@dataclass(frozen=True, slots=True) +class Skip: + message: str + + +ResourceAction = CreateReference | CreateVirtual | Skip diff --git a/packages/uipath/src/uipath/_cli/_push/_summary.py b/packages/uipath/src/uipath/_cli/_push/_summary.py new file mode 100644 index 000000000..2dcbca57e --- /dev/null +++ b/packages/uipath/src/uipath/_cli/_push/_summary.py @@ -0,0 +1,35 @@ +from dataclasses import dataclass + +import click + + +@dataclass +class ResourceImportSummary: + created: int = 0 + updated: int = 0 + unchanged: int = 0 + virtual_created: int = 0 + virtual_existing: int = 0 + not_found: int = 0 + + @property + def total(self) -> int: + return ( + self.created + + self.updated + + self.unchanged + + self.virtual_created + + self.virtual_existing + + self.not_found + ) + + def __str__(self) -> str: + return ( + f"\n \U0001f535 Resource import summary: {self.total} total resources - " + f"{click.style(str(self.created), fg='green')} created, " + f"{click.style(str(self.updated), fg='blue')} updated, " + f"{click.style(str(self.unchanged), fg='yellow')} unchanged, " + f"{click.style(str(self.virtual_created), fg='green')} virtual-created, " + f"{click.style(str(self.virtual_existing), fg='yellow')} virtual-existing, " + f"{click.style(str(self.not_found), fg='red')} not found" + ) diff --git a/packages/uipath/src/uipath/_cli/_push/_virtual_kinds.py b/packages/uipath/src/uipath/_cli/_push/_virtual_kinds.py new file mode 100644 index 000000000..bc1b20f73 --- /dev/null +++ b/packages/uipath/src/uipath/_cli/_push/_virtual_kinds.py @@ -0,0 +1,34 @@ +import logging + +from .._utils._studio_project import ResourceBuilderMetadataEntry, StudioClient + +logger = logging.getLogger(__name__) + +_FALLBACK: frozenset[str] = frozenset( + {"app", "asset", "bucket", "process", "queue", "taskCatalog", "trigger"} +) + + +async def fetch_supported_virtual_kinds(studio_client: StudioClient) -> set[str]: + """Return the set of resource kinds that support inline creation. + + Falls back to a static list on any failure — the caller shouldn't have to + care whether the metadata endpoint was reachable. + """ + try: + metadata = await studio_client.get_resource_builder_metadata() + except Exception as e: + logger.debug("Resource Builder metadata fetch failed, using fallback: %s", e) + return set(_FALLBACK) + return _extract_supported_kinds(metadata) + + +def _extract_supported_kinds( + metadata: list[ResourceBuilderMetadataEntry], +) -> set[str]: + # metadata has one entry per (kind, type), so a kind may appear multiple times + return { + entry.kind + for entry in metadata + if any(version.supports_in_line_creation for version in entry.versions) + } diff --git a/packages/uipath/src/uipath/_cli/_push/sw_file_handler.py b/packages/uipath/src/uipath/_cli/_push/sw_file_handler.py index 64a0e9b0e..19bfa060f 100644 --- a/packages/uipath/src/uipath/_cli/_push/sw_file_handler.py +++ b/packages/uipath/src/uipath/_cli/_push/sw_file_handler.py @@ -10,6 +10,12 @@ from uipath._cli.models.uipath_json_schema import PackOptions from uipath.platform.common import UiPathConfig +from uipath.platform.constants import ( + EVALS_FOLDER, + LEGACY_EVAL_FOLDER, + PYTHON_CONFIGURATION_FILE, + STUDIO_METADATA_FILE, +) from ...platform.errors import EnrichedException from .._utils._common import get_claim_from_token @@ -245,12 +251,12 @@ async def _process_file_uploads( deleted_files = self._collect_deleted_files( remote_files, processed_source_files, - files_to_ignore=["studio_metadata.json"], + files_to_ignore=[STUDIO_METADATA_FILE], directories_to_ignore=[ name for name, condition in [ - ("evals", not UiPathConfig.has_legacy_eval_folder), - ("evaluations", not UiPathConfig.has_eval_folder), + (LEGACY_EVAL_FOLDER, not UiPathConfig.has_legacy_eval_folder), + (EVALS_FOLDER, not UiPathConfig.has_eval_folder), ] if condition ], @@ -420,7 +426,7 @@ def get_author_from_token_or_toml() -> str: pass toml_data = read_toml_project( - os.path.join(self.directory, "pyproject.toml") + os.path.join(self.directory, PYTHON_CONFIGURATION_FILE) ) return toml_data.get("authors", "").strip() @@ -489,7 +495,7 @@ def get_author_from_token_or_toml() -> str: else: structural_migration.added_resources.append( AddedResource( - file_name="studio_metadata.json", + file_name=STUDIO_METADATA_FILE, content_string=json.dumps(metadata), parent_path=".uipath", ) @@ -543,7 +549,7 @@ async def upload_source_files( # Log skipped files from root evals folder if skipped_files: - evals_folder_path = os.path.join(self.directory, "evals") + evals_folder_path = os.path.join(self.directory, LEGACY_EVAL_FOLDER) logger.info( f"Skipping {len(skipped_files)} file(s) in evals folder ({evals_folder_path}): {', '.join(skipped_files)}" ) diff --git a/packages/uipath/src/uipath/_cli/_server_core.py b/packages/uipath/src/uipath/_cli/_server_core.py new file mode 100644 index 000000000..c426126ff --- /dev/null +++ b/packages/uipath/src/uipath/_cli/_server_core.py @@ -0,0 +1,108 @@ +"""Transport-agnostic job core shared by the HTTP and uipath-ipc channels.""" + +import asyncio +import os +import shlex +from typing import Any + +from .cli_debug import debug +from .cli_eval import eval +from .cli_run import run + +COMMANDS = { + "run": run, + "debug": debug, + "eval": eval, +} + + +class _ServerState: + """Mutable server state, initialized lazily at server startup.""" + + def __init__(self) -> None: + self.lock: asyncio.Lock | None = None + self.baseline_env: dict[str, str] | None = None + + def init(self) -> None: + """Must be called inside a running event loop at server startup.""" + if self.lock is not None: + return + self.lock = asyncio.Lock() + self.baseline_env = os.environ.copy() + + +_state = _ServerState() + + +def parse_args(args: str | list[str] | None) -> list[str]: + """Parse args into a list of strings.""" + if args is None: + return [] + if isinstance(args, list): + return args + if isinstance(args, str): + return shlex.split(args) + return [] + + +async def _run_command_isolated( + cmd: Any, + args: list[str], + env_vars: dict[str, str], + working_dir: str | None, +) -> dict[str, Any]: + """Run one command with per-job env/cwd isolation (the shared job core).""" + if _state.lock is None or _state.baseline_env is None: + raise RuntimeError("Server state not initialized") + + async with _state.lock: + original_cwd = os.getcwd() + try: + # Start from server baseline + request env vars only, so nothing from + # a previous job leaks through. + os.environ.clear() + os.environ.update(_state.baseline_env) + if isinstance(env_vars, dict): + os.environ.update(env_vars) + + if working_dir and isinstance(working_dir, str): + try: + os.chdir(working_dir) + except (FileNotFoundError, NotADirectoryError, PermissionError) as e: + # Request-shaped error: the caller gave a bad working dir. + # HTTP surfaces this as 400; IPC just returns ExitCode/Error. + return { + "ExitCode": 1, + "Error": f"Cannot change to working directory: {e}", + "Result": None, + "Unexpected": False, + "ClientError": True, + } + + result_value = await asyncio.to_thread( + cmd.main, args, standalone_mode=False + ) + return { + "ExitCode": 0, + "Error": None, + "Result": result_value, + "Unexpected": False, + } + except SystemExit as e: + exit_code = e.code if isinstance(e.code, int) else 1 + return { + "ExitCode": exit_code, + "Error": None if exit_code == 0 else f"Exit code: {exit_code}", + "Result": None, + "Unexpected": False, + } + except Exception as e: # report any job failure as a result, not a fault + return {"ExitCode": 1, "Error": str(e), "Result": None, "Unexpected": True} + finally: + # Restore to server baseline. + try: + os.chdir(original_cwd) + except OSError: + pass + os.environ.clear() + os.environ.update(_state.baseline_env) diff --git a/packages/uipath/src/uipath/_cli/_telemetry.py b/packages/uipath/src/uipath/_cli/_telemetry.py index 7adc54410..03d72a294 100644 --- a/packages/uipath/src/uipath/_cli/_telemetry.py +++ b/packages/uipath/src/uipath/_cli/_telemetry.py @@ -7,6 +7,7 @@ from uipath._cli._utils._common import get_claim_from_token from uipath.platform.common import UiPathConfig +from uipath.platform.constants import ENV_UIPATH_AGENT_ID from uipath.telemetry._track import ( _get_project_key, is_telemetry_enabled, @@ -41,28 +42,26 @@ def _enrich_properties(self, properties: Dict[str, Any]) -> None: Args: properties: The properties dictionary to enrich. """ - # Add UiPath context - project_key = _get_project_key() - if project_key: - properties["AgentId"] = project_key + agent_id = os.getenv(ENV_UIPATH_AGENT_ID) or _get_project_key() + if agent_id: + properties["AgentId"] = agent_id - # Get organization ID if UiPathConfig.organization_id: properties["CloudOrganizationId"] = UiPathConfig.organization_id - # Get tenant ID if UiPathConfig.tenant_id: properties["CloudTenantId"] = UiPathConfig.tenant_id - # Get CloudUserId from JWT token - try: - cloud_user_id = get_claim_from_token("sub") - if cloud_user_id: - properties["CloudUserId"] = cloud_user_id - except Exception: - pass + cloud_user_id = UiPathConfig.cloud_user_id + if not cloud_user_id: + try: + cloud_user_id = get_claim_from_token("sub") + except Exception: + cloud_user_id = None + if cloud_user_id: + properties["CloudUserId"] = cloud_user_id - properties["SessionId"] = "nosession" # Placeholder for session ID + properties["SessionId"] = "nosession" try: properties["SDKVersion"] = version("uipath") @@ -71,7 +70,6 @@ def _enrich_properties(self, properties: Dict[str, Any]) -> None: properties["IsGithubCI"] = bool(os.getenv("GITHUB_ACTIONS")) - # Add source identifier properties["Source"] = "uipath-python-cli" properties["ApplicationName"] = "UiPath.AgentCli" diff --git a/packages/uipath/src/uipath/_cli/_utils/_common.py b/packages/uipath/src/uipath/_cli/_utils/_common.py index 784192ab6..d346a5560 100644 --- a/packages/uipath/src/uipath/_cli/_utils/_common.py +++ b/packages/uipath/src/uipath/_cli/_utils/_common.py @@ -13,8 +13,8 @@ ResourceOverwriteParser, UiPathConfig, ) +from uipath.platform.constants import ENV_BASE_URL, ENV_UIPATH_ACCESS_TOKEN -from ..._utils.constants import ENV_UIPATH_ACCESS_TOKEN from ..models.runtime_schema import EntryPoint from ..spinner import Spinner from ._console import ConsoleLogger @@ -60,8 +60,8 @@ def environment_options(function): def get_env_vars(spinner: Spinner | None = None) -> list[str]: - base_url = os.environ.get("UIPATH_URL") - token = os.environ.get("UIPATH_ACCESS_TOKEN") + base_url = os.environ.get(ENV_BASE_URL) + token = os.environ.get(ENV_UIPATH_ACCESS_TOKEN) if not all([base_url, token]): if spinner: @@ -214,6 +214,14 @@ async def read_resource_overwrites_from_file( .get("internalArguments", {}) .get("resourceOverwrites", {}) ) + + logger.info( + "Resource overwrites read from %s (%d entries):\n%s", + file_path, + len(resource_overwrites), + json.dumps(resource_overwrites, indent=2, sort_keys=True), + ) + for key, value in resource_overwrites.items(): try: overwrites_dict[key] = ResourceOverwriteParser.parse(key, value) @@ -224,15 +232,9 @@ async def read_resource_overwrites_from_file( e, ) - logger.debug( - "Loaded %d resource overwrite(s) from file %s", - len(overwrites_dict), - file_path, - ) - # Return empty dict if file doesn't exist or invalid json except FileNotFoundError: - logger.debug("Resource overwrites config file not found: %s", file_path) + logger.info("Resource overwrites config file not found: %s", file_path) except json.JSONDecodeError as e: logger.warning("Failed to parse resource overwrites from %s: %s", file_path, e) diff --git a/packages/uipath/src/uipath/_cli/_utils/_constants.py b/packages/uipath/src/uipath/_cli/_utils/_constants.py index 14e2568b7..88dac9a96 100644 --- a/packages/uipath/src/uipath/_cli/_utils/_constants.py +++ b/packages/uipath/src/uipath/_cli/_utils/_constants.py @@ -20,7 +20,7 @@ DOCUMENT_EXTENSIONS = {".pdf", ".docx", ".pptx", ".xlsx", ".xls"} -ARCHIVE_EXTENSIONS = {".zip", ".tar", ".gz", ".rar", ".7z", ".bz2", ".xz"} +ARCHIVE_EXTENSIONS = {".zip", ".tar", ".gz", ".rar", ".7z", ".bz2", ".xz", ".whl"} MEDIA_EXTENSIONS = { ".mp3", diff --git a/packages/uipath/src/uipath/_cli/_utils/_project_files.py b/packages/uipath/src/uipath/_cli/_utils/_project_files.py index c7c025197..7121306a5 100644 --- a/packages/uipath/src/uipath/_cli/_utils/_project_files.py +++ b/packages/uipath/src/uipath/_cli/_utils/_project_files.py @@ -8,10 +8,16 @@ from pathlib import Path from typing import Any, AsyncIterator, Dict, Literal, Optional, Tuple +import anyio from pydantic import BaseModel, Field, TypeAdapter from uipath._cli.models.uipath_json_schema import PackOptions, UiPathJsonConfig from uipath.platform.common import UiPathConfig +from uipath.platform.constants import ( + LEGACY_EVAL_FOLDER, + PYTHON_CONFIGURATION_FILE, + UIPATH_CONFIG_FILE, +) from .._utils._console import ConsoleLogger from ._constants import is_binary_file @@ -25,6 +31,34 @@ logger = logging.getLogger(__name__) +def resolve_existing_project_id(directory: str = ".") -> Optional[str]: + """Return an already-established project id for this project, if any. + + Checks the Studio Web project env var first, then falls back to the legacy + ``ProjectKey`` stored in ``.uipath/.telemetry.json``. Returns ``None`` when + neither is present. + + Args: + directory: The project root directory to look for the telemetry file in. + """ + from ...telemetry._constants import _PROJECT_KEY, _TELEMETRY_CONFIG_FILE + + if project_id := UiPathConfig.project_id: + return project_id + + telemetry_file = os.path.join(directory, ".uipath", _TELEMETRY_CONFIG_FILE) + if os.path.exists(telemetry_file): + try: + with open(telemetry_file, "r") as f: + telemetry_data = json.load(f) + if project_id := telemetry_data.get(_PROJECT_KEY): + return project_id + except (json.JSONDecodeError, IOError): + pass + + return None + + class Severity(IntEnum): LOG = 0 WARNING = 1 @@ -89,8 +123,8 @@ def get_project_config(directory: str) -> dict[str, Any]: Raises: SystemExit: If required configuration files are missing or invalid """ - config_path = os.path.join(directory, "uipath.json") - toml_path = os.path.join(directory, "pyproject.toml") + config_path = os.path.join(directory, UIPATH_CONFIG_FILE) + toml_path = os.path.join(directory, PYTHON_CONFIGURATION_FILE) if not os.path.isfile(config_path): console.error("uipath.json not found, please run `uipath init`.") @@ -199,7 +233,7 @@ def ensure_config_file(directory: str) -> None: Raises: SystemExit: If uipath.json is not found in the directory """ - if not os.path.isfile(os.path.join(directory, "uipath.json")): + if not os.path.isfile(os.path.join(directory, UIPATH_CONFIG_FILE)): console.error( "uipath.json not found. Please run `uipath init` in the project directory." ) @@ -389,7 +423,7 @@ def files_to_include( tuple[list[FileInfo], list[str]]: Tuple of (included files, skipped file paths) """ file_extensions_included = [".py", ".mermaid", ".json", ".yaml", ".yml", ".md"] - files_included = ["pyproject.toml"] + files_included = [PYTHON_CONFIGURATION_FILE] files_excluded = [] if directories_to_ignore is None: @@ -429,7 +463,7 @@ def is_venv_dir(d: str) -> bool: # Determine if we're in the root evals folder root_rel_path = os.path.relpath(root, directory) normalized_root_rel_path = root_rel_path.replace(os.sep, "/") - is_root_evals_folder = normalized_root_rel_path == "evals" + is_root_evals_folder = normalized_root_rel_path == LEGACY_EVAL_FOLDER # Skip all directories that start with . or are a venv or are excluded included_dirs = [] @@ -592,21 +626,21 @@ async def download_folder_files( collect_files_from_folder(folder, "", files_dict) for file_path, remote_file in files_dict.items(): - local_path = base_path / file_path - local_path.parent.mkdir(parents=True, exist_ok=True) + local_path = anyio.Path(base_path / file_path) + await local_path.parent.mkdir(parents=True, exist_ok=True) response = await studio_client.download_project_file_async(remote_file) remote_content = response.read().decode("utf-8") remote_hash = compute_normalized_hash(remote_content) - if os.path.exists(local_path): - with open(local_path, "r", encoding="utf-8") as f: - local_content = f.read() - local_hash = compute_normalized_hash(local_content) + if await local_path.exists(): + local_content = await local_path.read_text(encoding="utf-8") + local_hash = compute_normalized_hash(local_content) if local_hash != remote_hash: - with open(local_path, "w", encoding="utf-8", newline="\n") as f: - f.write(remote_content) + await local_path.write_text( + remote_content, encoding="utf-8", newline="\n" + ) yield UpdateEvent( file_path=file_path, @@ -620,8 +654,7 @@ async def download_folder_files( message=f"File '{file_path}' is up to date", ) else: - with open(local_path, "w", encoding="utf-8", newline="\n") as f: - f.write(remote_content) + await local_path.write_text(remote_content, encoding="utf-8", newline="\n") yield UpdateEvent( file_path=file_path, diff --git a/packages/uipath/src/uipath/_cli/_utils/_service_base.py b/packages/uipath/src/uipath/_cli/_utils/_service_base.py index 018e2e112..6441b573e 100644 --- a/packages/uipath/src/uipath/_cli/_utils/_service_base.py +++ b/packages/uipath/src/uipath/_cli/_utils/_service_base.py @@ -19,6 +19,8 @@ import click from httpx import HTTPError +from uipath.platform.constants import ENV_BASE_URL, ENV_UIPATH_ACCESS_TOKEN + from ...platform.errors import ( BaseUrlMissingError, EnrichedException, @@ -321,8 +323,8 @@ def get_client(ctx): if cli_ctx._client is None: from ...platform._uipath import UiPath - base_url = os.environ.get("UIPATH_URL") - secret = os.environ.get("UIPATH_ACCESS_TOKEN") + base_url = os.environ.get(ENV_BASE_URL) + secret = os.environ.get(ENV_UIPATH_ACCESS_TOKEN) if not base_url: raise click.ClickException( diff --git a/packages/uipath/src/uipath/_cli/_utils/_studio_project.py b/packages/uipath/src/uipath/_cli/_utils/_studio_project.py index 63ddceb37..10eccdf5d 100644 --- a/packages/uipath/src/uipath/_cli/_utils/_studio_project.py +++ b/packages/uipath/src/uipath/_cli/_utils/_studio_project.py @@ -6,22 +6,21 @@ from pathlib import PurePath from typing import Any, Callable, List, Optional, Union -import click from pydantic import BaseModel, ConfigDict, Field, field_validator -from uipath._utils.constants import ( - ENV_TENANT_ID, - HEADER_SW_LOCK_KEY, - HEADER_TENANT_ID, - PYTHON_CONFIGURATION_FILE, - STUDIO_METADATA_FILE, -) from uipath.platform import UiPath from uipath.platform.common import ( ResourceOverwrite, ResourceOverwriteParser, UiPathConfig, ) +from uipath.platform.constants import ( + ENV_TENANT_ID, + HEADER_SW_LOCK_KEY, + HEADER_TENANT_ID, + PYTHON_CONFIGURATION_FILE, + STUDIO_METADATA_FILE, +) from uipath.platform.errors import EnrichedException from uipath.tracing import traced @@ -152,12 +151,20 @@ class LockInfo(BaseModel): solution_lock_key: Optional[str] = Field(alias="solutionLockKey") -class Severity(str, Enum): - """Severity level for virtual resource operation results.""" +class ResourceBuilderMetadataVersion(BaseModel): + model_config = ConfigDict(extra="allow") + + supports_in_line_creation: bool = Field( + default=False, alias="supportsInLineCreation" + ) + + +class ResourceBuilderMetadataEntry(BaseModel): + model_config = ConfigDict(extra="allow") - SUCCESS = "success" - ATTENTION = "attention" - WARN = "warn" + kind: str + type: str | None = None + versions: list[ResourceBuilderMetadataVersion] = Field(default_factory=list) class VirtualResourceRequest(BaseModel): @@ -173,16 +180,20 @@ class VirtualResourceRequest(BaseModel): api_version: Optional[str] = Field(default=None, alias="apiVersion") +class Status(str, Enum): + ADDED = "ADDED" + UNCHANGED = "UNCHANGED" + UPDATED = "UPDATED" + + class VirtualResourceResult(BaseModel): - """Result of a virtual resource creation operation. + """Structured outcome of a virtual resource creation attempt. - Attributes: - severity: The severity level (log, warn or attention) - message: The result message with styling + Only `ADDED` and `UNCHANGED` are possible — virtual resources are never + updated in place. """ - severity: Severity - message: str + status: Status class ReferencedResourceFolder(BaseModel): @@ -362,12 +373,6 @@ class ProjectLockUnavailableError(RuntimeError): pass -class Status(str, Enum): - ADDED = "ADDED" - UNCHANGED = "UNCHANGED" - UPDATED = "UPDATED" - - class ReferencedResourceResponse(BaseModel): """Response from creating a referenced resource. @@ -508,6 +513,9 @@ async def _get_solution_id(self) -> str: async def ensure_coded_agent_project_async(self): structure = await self.get_project_structure_async() + # An empty structure means a never-pushed project: allow the first push. + if not structure.files and not structure.folders: + return if not any(file.name == PYTHON_CONFIGURATION_FILE for file in structure.files): raise NonCodedAgentProjectException() @@ -528,6 +536,19 @@ async def get_project_metadata_async(self) -> Optional[StudioProjectMetadata]: response.read().decode("utf-8") ) + async def get_resource_builder_metadata( + self, + ) -> list[ResourceBuilderMetadataEntry]: + response = await self.uipath.api_client.request_async( + "GET", + url="/studio_/backend/api/resourcebuilder/metadata", + scoped="org", + ) + return [ + ResourceBuilderMetadataEntry.model_validate(entry) + for entry in response.json() + ] + async def _get_existing_resources(self) -> List[dict[str, Any]]: if self._resources_cache is not None: return self._resources_cache @@ -560,6 +581,12 @@ async def get_resource_overwrites(self) -> dict[str, ResourceOverwrite]: with open(UiPathConfig.bindings_file_path, "rb") as f: file_content = f.read() + logger.info( + "Resource bindings (%s):\n%s", + UiPathConfig.bindings_file_path, + file_content.decode(), + ) + solution_id = await self._get_solution_id() tenant_id = os.getenv(ENV_TENANT_ID, None) @@ -582,85 +609,36 @@ async def get_resource_overwrites(self) -> dict[str, ResourceOverwrite]: files=files, ) data = response.json() - overwrites = {} - - for key, value in data.items(): - overwrites[key] = ResourceOverwriteParser.parse(key, value) logger.info( - "Loaded %d resource overwrite(s) from Studio API for solution %s: %s", - len(overwrites), + "Resource overwrites received for solution %s (%d entries):\n%s", solution_id, - overwrites, + len(data), + json.dumps(data, indent=2), ) + overwrites = {} + for key, value in data.items(): + overwrites[key] = ResourceOverwriteParser.parse(key, value) + return overwrites async def create_virtual_resource( self, virtual_resource_request: VirtualResourceRequest ) -> VirtualResourceResult: - """Create a virtual resource or return appropriate status if it already exists. - - Args: - virtual_resource_request: The virtual resource request details + """Create a virtual resource, or report UNCHANGED if already present. - Returns: - VirtualResourceResult: Result indicating the operation status and a formatted message + Returns UNCHANGED when the same name+kind already exists in the + solution. Name collisions with a different kind are not checked + client-side — they surface as a server error via EnrichedException. """ - # Build base message with resource details - base_message_parts = [ - f"Resource {click.style(virtual_resource_request.name, fg='cyan')}", - f" (kind: {click.style(virtual_resource_request.kind, fg='yellow')}", - ] - - if virtual_resource_request.type: - base_message_parts.append( - f", type: {click.style(virtual_resource_request.type, fg='yellow')}" - ) - - if virtual_resource_request.activity_name: - base_message_parts.append( - f", activity: {click.style(virtual_resource_request.activity_name, fg='yellow')}" - ) - - base_message_parts.append(")") - base_message = "".join(base_message_parts) - + name = virtual_resource_request.name + kind = virtual_resource_request.kind existing_resources = await self._get_existing_resources() - # Check if resource with same kind and name exists - existing_same_kind = next( - ( - r - for r in existing_resources - if r["name"] == virtual_resource_request.name - and r["kind"] == virtual_resource_request.kind - ), - None, - ) - if existing_same_kind: - message = f"{base_message} already exists. Skipping..." - return VirtualResourceResult(severity=Severity.ATTENTION, message=message) - - # Check if resource with same name but different kind exists - existing_diff_kind = next( - ( - r - for r in existing_resources - if r["name"] == virtual_resource_request.name - and r["kind"] != virtual_resource_request.kind - ), - None, - ) - if existing_diff_kind: - message = ( - f"Cannot create {base_message}. " - f"A resource with this name already exists with kind {click.style(existing_diff_kind['kind'], fg='yellow')}. " - f"Consider renaming the resource in code." - ) - return VirtualResourceResult(severity=Severity.WARN, message=message) + if any(r["name"] == name and r["kind"] == kind for r in existing_resources): + return VirtualResourceResult(status=Status.UNCHANGED) - # Create the virtual resource solution_id = await self._get_solution_id() response = await self.uipath.api_client.request_async( "POST", @@ -669,21 +647,12 @@ async def create_virtual_resource( json=virtual_resource_request.model_dump(exclude_none=True), ) resource_key = response.json()["key"] - await self._update_resource_specs( - resource_key, new_specs={"name": virtual_resource_request.name} - ) + await self._update_resource_specs(resource_key, new_specs={"name": name}) - # Update cache with newly created resource if self._resources_cache is not None: - self._resources_cache.append( - { - "name": virtual_resource_request.name, - "kind": virtual_resource_request.kind, - } - ) + self._resources_cache.append({"name": name, "kind": kind}) - message = f"{base_message} created successfully." - return VirtualResourceResult(severity=Severity.SUCCESS, message=message) + return VirtualResourceResult(status=Status.ADDED) async def create_referenced_resource( self, referenced_resource_request: ReferencedResourceRequest @@ -744,13 +713,22 @@ async def get_project_structure_async( if not force and self._project_structure_cache is not None: return self._project_structure_cache - response = await self.uipath.api_client.request_async( - "GET", - url=f"{self.file_operations_base_url}/Structure", - scoped="org", - ) - - self._project_structure_cache = ProjectStructure.model_validate(response.json()) + try: + response = await self.uipath.api_client.request_async( + "GET", + url=f"{self.file_operations_base_url}/Structure", + scoped="org", + ) + structure = ProjectStructure.model_validate(response.json()) + except EnrichedException as e: + # The backend returns 404 for projects whose file system was never + # initialized (e.g. a freshly created Function project): treat it + # as an empty structure so the first push can bootstrap the files. + if e.status_code != 404: + raise + structure = ProjectStructure(name="root", folders=[], files=[]) + + self._project_structure_cache = structure return self._project_structure_cache @traced(name="create_folder", run_type="uipath") diff --git a/packages/uipath/src/uipath/_cli/_utils/_tracing.py b/packages/uipath/src/uipath/_cli/_utils/_tracing.py index fdc4a2238..b085075f0 100644 --- a/packages/uipath/src/uipath/_cli/_utils/_tracing.py +++ b/packages/uipath/src/uipath/_cli/_utils/_tracing.py @@ -24,6 +24,15 @@ def filter(self, record): return False +def create_trace_manager(): + from uipath.core.tracing import UiPathTraceManager + from uipath.platform.common import ReferenceHierarchySpanProcessor + + tm = UiPathTraceManager() + tm.add_span_processor(ReferenceHierarchySpanProcessor()) + return tm + + def setup_tracer_httpx_logging(url: str): # Create a custom logger for httpx # Add the custom filter to the root logger diff --git a/packages/uipath/src/uipath/_cli/cli_debug.py b/packages/uipath/src/uipath/_cli/cli_debug.py index 44415dd26..7d7eceba2 100644 --- a/packages/uipath/src/uipath/_cli/cli_debug.py +++ b/packages/uipath/src/uipath/_cli/cli_debug.py @@ -1,15 +1,22 @@ import asyncio import logging +from typing import Any, cast, get_args import click +from pydantic import ValidationError from uipath._cli._chat._bridge import get_chat_bridge -from uipath._cli._debug._bridge import get_debug_bridge +from uipath._cli._debug._bridge import DebugAttachMode, get_debug_bridge from uipath._cli._utils._debug import setup_debugging from uipath._cli._utils._studio_project import StudioClient -from uipath.core.tracing import UiPathTraceManager -from uipath.eval.mocks import UiPathMockRuntime -from uipath.platform.common import ResourceOverwritesContext, UiPathConfig +from uipath._cli._utils._tracing import create_trace_manager +from uipath.eval.mocks import SimulationConfig, UiPathMockRuntime, build_mocking_context +from uipath.eval.mocks._mock_runtime import load_simulation_config +from uipath.platform.common import ( + ExecutionSourceContext, + ResourceOverwritesContext, + UiPathConfig, +) from uipath.runtime import ( UiPathExecuteOptions, UiPathRuntimeContext, @@ -21,6 +28,7 @@ from uipath.runtime.debug import UiPathDebugProtocol, UiPathDebugRuntime from uipath.tracing import LiveTrackingSpanProcessor, LlmOpsHttpExporter +from ._governance_bootstrap import GovernanceBootstrap, resolve_governance from ._telemetry import track_command from ._utils._console import ConsoleLogger from .middlewares import Middlewares @@ -63,6 +71,21 @@ default=5678, help="Port for the debug server (default: 5678)", ) +@click.option( + "--attach", + type=click.Choice(list(get_args(DebugAttachMode)), case_sensitive=False), + default=None, + help=( + "Debugger attach mode. Defaults to 'signalr' for cloud runs, " + "'console' for local runs." + ), +) +@click.option( + "--simulation", + required=False, + default=None, + help="Simulation config as a JSON object (same schema as simulation.json)", +) @track_command("debug") def debug( entrypoint: str | None, @@ -73,6 +96,8 @@ def debug( output_file: str | None, debug: bool, debug_port: int, + attach: str | None, + simulation: str | None, ) -> None: """Debug the project.""" input_file = file or input_file @@ -80,6 +105,18 @@ def debug( if not setup_debugging(debug, debug_port): console.error(f"Failed to start debug server on port {debug_port}") + simulation_config: SimulationConfig | None = None + if simulation: + try: + simulation_config = SimulationConfig.model_validate_json(simulation) + except (ValidationError, ValueError) as e: + console.error(f"Invalid --simulation config: {e}") + return + + attach_mode: DebugAttachMode | None = ( + cast(DebugAttachMode, attach.lower()) if attach else None + ) + result = Middlewares.next( "debug", entrypoint, @@ -89,6 +126,7 @@ def debug( output_file=output_file, debug=debug, debug_port=debug_port, + attach=attach_mode, ) if result.error_message: @@ -103,17 +141,19 @@ def debug( try: async def execute_debug_runtime(): - trace_manager = UiPathTraceManager() + trace_manager = create_trace_manager() - with UiPathRuntimeContext.with_defaults( + ctx = UiPathRuntimeContext.with_defaults( input=input, input_file=input_file, output_file=output_file, resume=resume, trace_manager=trace_manager, command="debug", - ) as ctx: + ) + with ExecutionSourceContext(ctx.execution_source), ctx: factory: UiPathRuntimeFactoryProtocol | None = None + governance_bootstrap: GovernanceBootstrap | None = None try: trigger_poll_interval: float = 5.0 @@ -125,6 +165,22 @@ async def execute_debug_runtime(): if factory_settings else None ) + agent_type = ( + factory_settings.agent_type if factory_settings else None + ) + agent_framework = ( + factory_settings.agent_framework + if factory_settings + else None + ) + governance_bootstrap = await resolve_governance( + agent_framework=agent_framework, + agent_type=agent_type, + is_conversational=ctx.conversation_id is not None, + ) + governance_runtime_id = ( + ctx.conversation_id or ctx.job_id or "default" + ) if ctx.job_id: if UiPathConfig.is_tracing_enabled: @@ -140,13 +196,27 @@ async def execute_debug_runtime(): async def execute_debug_runtime(): chat_runtime: UiPathRuntimeProtocol | None = None - debug_bridge: UiPathDebugProtocol = get_debug_bridge(ctx) - + debug_bridge: UiPathDebugProtocol = get_debug_bridge( + ctx, attach=attach_mode + ) + new_runtime_kwargs: dict[str, Any] = {} + if governance_bootstrap is not None: + new_runtime_kwargs["evaluator"] = ( + governance_bootstrap.evaluator + ) runtime = await factory.new_runtime( entrypoint, - ctx.conversation_id or ctx.job_id or "default", + governance_runtime_id, + **new_runtime_kwargs, ) + if governance_bootstrap is not None: + runtime = governance_bootstrap.wrap_runtime( + runtime, + agent_name=entrypoint, + runtime_id=governance_runtime_id, + ) + delegate = runtime if ctx.conversation_id and ctx.exchange_id: chat_bridge: UiPathChatProtocol = get_chat_bridge( @@ -163,17 +233,41 @@ async def execute_debug_runtime(): trigger_poll_interval=trigger_poll_interval, ) - mock_runtime = UiPathMockRuntime( - delegate=debug_runtime, + # Build mocking context with agent model for simulations + schema = await runtime.get_schema() + agent_model = None + if schema.metadata and "settings" in schema.metadata: + agent_model = schema.metadata["settings"].get("model") + + delegate_runtime: UiPathDebugRuntime | UiPathMockRuntime = ( + debug_runtime ) + if simulation_config: + mocking_context = build_mocking_context( + simulation_config, agent_model + ) + if mocking_context: + delegate_runtime = UiPathMockRuntime( + delegate=debug_runtime, + mocking_context=mocking_context, + ) + else: + mocking_context = load_simulation_config( + agent_model=agent_model + ) + delegate_runtime = UiPathMockRuntime( + delegate=debug_runtime, + mocking_context=mocking_context, + ) try: - ctx.result = await mock_runtime.execute( + ctx.result = await delegate_runtime.execute( ctx.get_input(), options=UiPathExecuteOptions(resume=resume), ) finally: - await mock_runtime.dispose() + if delegate_runtime is not debug_runtime: + await delegate_runtime.dispose() await debug_runtime.dispose() if chat_runtime: await chat_runtime.dispose() @@ -193,8 +287,13 @@ async def execute_debug_runtime(): await execute_debug_runtime() finally: - if factory: - await factory.dispose() + try: + if governance_bootstrap is not None: + governance_bootstrap.dispose() + if factory: + await factory.dispose() + finally: + trace_manager.shutdown() asyncio.run(execute_debug_runtime()) except Exception as e: diff --git a/packages/uipath/src/uipath/_cli/cli_dev.py b/packages/uipath/src/uipath/_cli/cli_dev.py index 62740dc4b..16b331dd3 100644 --- a/packages/uipath/src/uipath/_cli/cli_dev.py +++ b/packages/uipath/src/uipath/_cli/cli_dev.py @@ -5,8 +5,10 @@ from uipath._cli._utils._console import ConsoleLogger from uipath._cli._utils._debug import setup_debugging +from uipath._cli._utils._tracing import create_trace_manager from uipath._cli.middlewares import Middlewares from uipath.core.tracing import UiPathTraceManager +from uipath.platform.common import ExecutionSourceContext from uipath.runtime import UiPathRuntimeContext, UiPathRuntimeFactoryRegistry from ._telemetry import track_command @@ -14,6 +16,12 @@ console = ConsoleLogger() +def _create_dev_context_and_factory(trace_manager: UiPathTraceManager): + """Build the dev runtime context and its factory.""" + context = UiPathRuntimeContext(trace_manager=trace_manager, command="dev") + return context, UiPathRuntimeFactoryRegistry.get(context=context) + + def _check_dev_dependency(interface: str) -> None: """Check if uipath-dev is installed and raise helpful error if not.""" import importlib.util @@ -79,18 +87,15 @@ async def run_terminal() -> None: factory = None try: - trace_manager = UiPathTraceManager() - factory = UiPathRuntimeFactoryRegistry.get( - context=UiPathRuntimeContext( - trace_manager=trace_manager, command="dev" - ) - ) + trace_manager = create_trace_manager() + context, factory = _create_dev_context_and_factory(trace_manager) app = UiPathDeveloperConsole( runtime_factory=factory, trace_manager=trace_manager ) - await app.run_async() + with ExecutionSourceContext(context.execution_source): + await app.run_async() except KeyboardInterrupt: console.info("Debug session interrupted by user") @@ -123,12 +128,8 @@ def signal_handler(sig, frame): signal.signal(signal.SIGTERM, signal_handler) try: - trace_manager = UiPathTraceManager() - factory = UiPathRuntimeFactoryRegistry.get( - context=UiPathRuntimeContext( - trace_manager=trace_manager, command="dev" - ) - ) + trace_manager = create_trace_manager() + context, factory = _create_dev_context_and_factory(trace_manager) app = UiPathDeveloperServer( runtime_factory=factory, @@ -140,13 +141,17 @@ def signal_handler(sig, frame): ), ) - server_task = asyncio.create_task(app.run_async()) - shutdown_task = asyncio.create_task(shutdown_event.wait()) + # Enter the execution source context before creating the server + # task so request tasks spawned during the run inherit it. + with ExecutionSourceContext(context.execution_source): + server_task = asyncio.create_task(app.run_async()) + shutdown_task = asyncio.create_task(shutdown_event.wait()) - # Wait for either server to complete or shutdown signal - done, pending = await asyncio.wait( - {server_task, shutdown_task}, return_when=asyncio.FIRST_COMPLETED - ) + # Wait for either server to complete or shutdown signal + done, pending = await asyncio.wait( + {server_task, shutdown_task}, + return_when=asyncio.FIRST_COMPLETED, + ) for task in pending: task.cancel() diff --git a/packages/uipath/src/uipath/_cli/cli_eval.py b/packages/uipath/src/uipath/_cli/cli_eval.py index d0bdc730c..66bdfad10 100644 --- a/packages/uipath/src/uipath/_cli/cli_eval.py +++ b/packages/uipath/src/uipath/_cli/cli_eval.py @@ -3,28 +3,34 @@ import logging import os import uuid +from contextlib import AsyncExitStack from pathlib import Path from typing import Any import click +from uipath._cli._errors import EntrypointDiscoveryException from uipath._cli._evals._console_progress_reporter import ConsoleProgressReporter from uipath._cli._evals._progress_reporter import StudioWebProgressReporter from uipath._cli._evals._telemetry import EvalTelemetrySubscriber from uipath._cli._utils._folders import get_personal_workspace_key_async from uipath._cli._utils._studio_project import StudioClient +from uipath._cli._utils._tracing import create_trace_manager from uipath._cli.middlewares import Middlewares from uipath.core.events import EventBus -from uipath.core.tracing import UiPathTraceManager -from uipath.eval.helpers import EVAL_SETS_DIRECTORY_NAME, EvalHelpers +from uipath.eval.helpers import EVAL_SETS_DIRECTORY_NAME, EvalHelpers, get_agent_model from uipath.eval.models.evaluation_set import EvaluationSet from uipath.eval.runtime import UiPathEvalContext, evaluate from uipath.platform.chat import set_llm_concurrency -from uipath.platform.common import ResourceOverwritesContext, UiPathConfig +from uipath.platform.common import ( + ExecutionSourceContext, + ResourceOverwritesContext, + UiPathConfig, +) +from uipath.platform.constants import ENV_FOLDER_KEY from uipath.runtime import ( UiPathRuntimeContext, UiPathRuntimeFactoryRegistry, - UiPathRuntimeSchema, ) from uipath.telemetry._track import flush_events from uipath.tracing import ( @@ -60,31 +66,10 @@ def setup_reporting_prereq(no_report: bool) -> bool: if not UiPathConfig.folder_key: folder_key = asyncio.run(get_personal_workspace_key_async()) if folder_key: - os.environ["UIPATH_FOLDER_KEY"] = folder_key + os.environ[ENV_FOLDER_KEY] = folder_key return True -def _get_agent_model(schema: UiPathRuntimeSchema) -> str | None: - """Get agent model from the runtime schema metadata. - - The model is read from schema.metadata["settings"]["model"] which is - populated by the low-code agents runtime from agent.json. - - Returns: - The model name from agent settings, or None if not found. - """ - try: - if schema.metadata and "settings" in schema.metadata: - settings = schema.metadata["settings"] - model = settings.get("model") - if model: - logger.debug(f"Got agent model from schema.metadata: {model}") - return model - return None - except Exception: - return None - - def _resolve_model_settings_override( model_settings_id: str, evaluation_set: EvaluationSet ) -> dict[str, Any] | None: @@ -135,13 +120,96 @@ def _resolve_model_settings_override( return override if override else None -class _EvalDiscoveryError(Exception): +def _resolve_agent_memory_settings_override( + agent_memory_settings_id: str, evaluation_set: EvaluationSet +) -> dict[str, Any]: + """Resolve agent memory settings override from evaluation set. + + Evaluation runs never consume agent memory unless the evaluation set + explicitly enables it (agentMemoryEnabled), mirroring the Agents + backend behavior for eval runs. + + Returns: + Memory override dict passed to the factory via the + agent_memory_settings kwarg. ``{"enabled": False}`` disables agent + memory for the run; ``{"enabled": True, ...}`` force-enables it with + the selected settings ("same-as-agent" values preserve the agent's + own configuration). + """ + if not evaluation_set.agent_memory_enabled: + return {"enabled": False} + + # "NoMemory" is a sentinel id meaning "run without memory", matching the + # Agents backend (ApplyAgentMemorySettingsOverride removes the memorySpace + # feature for a null or "NoMemory" setting). Its entry's field values are + # also "NoMemory" strings, so it must never be applied as real settings. + if agent_memory_settings_id == "NoMemory": + return {"enabled": False} + + memory_settings = evaluation_set.agent_memory_settings + target = None + if agent_memory_settings_id: + # "default" is looked up like any other id: the eval-set editor persists a + # "default" entry (all fields "same-as-agent") alongside user-defined ones. + target = next( + (ms for ms in memory_settings if ms.id == agent_memory_settings_id), + None, + ) + if not target and agent_memory_settings_id != "default": + logger.warning( + f"Agent memory settings ID '{agent_memory_settings_id}' not found in evaluation set" + ) + if target is None: + target = memory_settings[0] if memory_settings else None + + if target is not None and target.id == "NoMemory": + return {"enabled": False} + + if target is None: + # Memory enabled but no settings configured: keep the agent's own configuration + return {"enabled": True} + + logger.info( + f"Applying agent memory settings override: searchMode={target.search_mode}, " + f"resultCount={target.result_count}, threshold={target.threshold}" + ) + return { + "enabled": True, + "resultCount": target.result_count, + "searchMode": target.search_mode, + "threshold": target.threshold, + } + + +class _EvalDiscoveryError(EntrypointDiscoveryException): """Raised when auto-discovery of entrypoint or eval set fails.""" def __init__(self, entrypoints: list[str], eval_sets: list[Path]): - self.entrypoints = entrypoints + super().__init__(entrypoints) self.eval_sets = eval_sets + def get_usage_help(self) -> list[str]: + lines = super().get_usage_help() + + if self.eval_sets: + lines.append("") + lines.append("Available eval sets:") + for f in self.eval_sets: + lines.append(f" - {f}") + else: + lines.append("") + lines.append( + f"No eval sets found in '{EVAL_SETS_DIRECTORY_NAME}/' directory." + ) + + lines.append("") + lines.append("Usage: uipath eval ") + if self.entrypoints and self.eval_sets: + lines.append( + f"Example: uipath eval {self.entrypoints[0]} {self.eval_sets[0]}" + ) + return lines + def _discover_eval_sets() -> list[Path]: """Discover available eval set files.""" @@ -151,39 +219,6 @@ def _discover_eval_sets() -> list[Path]: return [] -def _show_eval_usage_help(entrypoints: list[str], eval_set_files: list[Path]) -> None: - """Show available entrypoints and eval sets with usage examples.""" - lines: list[str] = [] - - if entrypoints: - lines.append("Available entrypoints:") - for name in entrypoints: - lines.append(f" - {name}") - else: - lines.append( - "No entrypoints found. " - "Add a 'functions' or 'agents' section to your config file " - "(e.g. uipath.json, langgraph.json)." - ) - - if eval_set_files: - lines.append("\nAvailable eval sets:") - for f in eval_set_files: - lines.append(f" - {f}") - else: - lines.append( - f"\nNo eval sets found in '{EVAL_SETS_DIRECTORY_NAME}/' directory." - ) - - lines.append("\nUsage: uipath eval ") - if entrypoints and eval_set_files: - ep_name = entrypoints[0] - es_path = eval_set_files[0] - lines.append(f"Example: uipath eval {ep_name} {es_path}") - - click.echo("\n".join(lines)) - - @click.command() @click.argument("entrypoint", required=False) @click.argument("eval_set", required=False) @@ -230,6 +265,12 @@ def _show_eval_usage_help(entrypoints: list[str], eval_set_files: list[Path]) -> default="default", help="Model settings ID from evaluation set to override agent settings (default: 'default')", ) +@click.option( + "--agent-memory-settings-id", + type=str, + default="default", + help="Agent memory settings ID from evaluation set to override agent memory settings (default: 'default')", +) @click.option( "--trace-file", required=False, @@ -258,7 +299,7 @@ def _show_eval_usage_help(entrypoints: list[str], eval_set_files: list[Path]) -> "--verbose", is_flag=True, default=False, - help="Include agent execution output (trace, result) in the output file", + help="Include workload execution output (trace, result) in the output file", ) def eval( entrypoint: str | None, @@ -271,6 +312,7 @@ def eval( enable_mocker_cache: bool, report_coverage: bool, model_settings_id: str, + agent_memory_settings_id: str, trace_file: str | None, max_llm_concurrency: int, input_overrides: dict[str, Any], @@ -289,6 +331,7 @@ def eval( enable_mocker_cache: Enable caching for LLM mocker responses report_coverage: Report evaluation coverage model_settings_id: Model settings ID to override agent settings + agent_memory_settings_id: Agent memory settings ID to override agent memory settings trace_file: File path where traces will be written in JSONL format max_llm_concurrency: Maximum concurrent LLM requests input_overrides: Input field overrides mapping (direct field override with deep merge) @@ -339,14 +382,15 @@ async def execute_eval(): telemetry_subscriber = EvalTelemetrySubscriber() await telemetry_subscriber.subscribe_to_eval_runtime_events(event_bus) - trace_manager = UiPathTraceManager() + trace_manager = create_trace_manager() - with UiPathRuntimeContext.with_defaults( + ctx = UiPathRuntimeContext.with_defaults( output_file=output_file, trace_manager=trace_manager, command="eval", resume=resume, - ) as ctx: + ) + with ExecutionSourceContext(ctx.execution_source), ctx: # Set job_id in eval context for single runtime runs eval_context.job_id = ctx.job_id @@ -429,40 +473,52 @@ async def execute_eval(): settings_override = _resolve_model_settings_override( model_settings_id, eval_context.evaluation_set ) - - runtime = await runtime_factory.new_runtime( - entrypoint=eval_context.entrypoint or "", - runtime_id=eval_context.execution_id, - settings=settings_override, - ) - - eval_context.runtime_schema = await runtime.get_schema() - - eval_context.evaluators = await EvalHelpers.load_evaluators( - resolved_eval_set_path, - eval_context.evaluation_set, - _get_agent_model(eval_context.runtime_schema), + agent_memory_settings_override = ( + _resolve_agent_memory_settings_override( + agent_memory_settings_id, eval_context.evaluation_set + ) ) - # Runtime is not required anymore. - await runtime.dispose() + # Resource overwrites must be in scope before any runtime is + # created: building the agent graph resolves folder-scoped + # resources (e.g. escalation memory spaces) at tool-creation + # time, and those lookups need the overwritten folder paths. + async with AsyncExitStack() as stack: + if project_id: + studio_client = StudioClient(project_id) + + await stack.enter_async_context( + ResourceOverwritesContext( + lambda: studio_client.get_resource_overwrites() + ) + ) + else: + logger.debug( + "No UIPATH_PROJECT_ID configured, executing evaluation without resource overwrites" + ) - if project_id: - studio_client = StudioClient(project_id) + runtime = await runtime_factory.new_runtime( + entrypoint=eval_context.entrypoint or "", + runtime_id=eval_context.execution_id, + settings=settings_override, + agent_memory_settings=agent_memory_settings_override, + ) - async with ResourceOverwritesContext( - lambda: studio_client.get_resource_overwrites() - ): - ctx.result = await evaluate( - runtime_factory, - trace_manager, - eval_context, - event_bus, + # The runtime is only needed for schema/evaluator + # loading; dispose it before evaluation starts. + try: + eval_context.runtime_schema = await runtime.get_schema() + + eval_context.evaluators = ( + await EvalHelpers.load_evaluators( + resolved_eval_set_path, + eval_context.evaluation_set, + get_agent_model(eval_context.runtime_schema), + ) ) - else: - logger.debug( - "No UIPATH_PROJECT_ID configured, executing evaluation without resource overwrites" - ) + finally: + await runtime.dispose() + ctx.result = await evaluate( runtime_factory, trace_manager, @@ -475,7 +531,13 @@ async def execute_eval(): asyncio.run(execute_eval()) except _EvalDiscoveryError as e: - _show_eval_usage_help(e.entrypoints, e.eval_sets) + click.echo("\n".join(e.get_usage_help())) + if not e.entrypoints: + click.echo() + console.link( + "uipath.json spec:", + "https://github.com/UiPath/uipath-python/blob/main/packages/uipath/specs/uipath.spec.md", + ) except ValueError as e: console.error(str(e)) except Exception as e: diff --git a/packages/uipath/src/uipath/_cli/cli_init.py b/packages/uipath/src/uipath/_cli/cli_init.py index f86e30f06..d6ff01a90 100644 --- a/packages/uipath/src/uipath/_cli/cli_init.py +++ b/packages/uipath/src/uipath/_cli/cli_init.py @@ -9,6 +9,7 @@ from pathlib import Path from typing import Any +import anyio import click from graphtty import RenderOptions, render from graphtty.themes import TOKYO_NIGHT @@ -22,6 +23,12 @@ ) from uipath.platform.common import UiPathConfig +from uipath.platform.constants import ( + DOTENV_FILE, + ENTRY_POINTS_FILE, + PYTHON_CONFIGURATION_FILE, + UIPATH_CONFIG_FILE, +) from uipath.runtime import ( UiPathRuntimeContext, UiPathRuntimeFactoryProtocol, @@ -30,13 +37,11 @@ ) from uipath.runtime.schema import UiPathRuntimeGraph, UiPathRuntimeSchema -from .._utils.constants import ENV_TELEMETRY_ENABLED -from ..telemetry._constants import _PROJECT_KEY, _TELEMETRY_CONFIG_FILE from ._telemetry import track_command from ._utils._common import determine_project_type from ._utils._console import ConsoleLogger from ._utils._constants import AGENT_INITIAL_CODE_VERSION, SCHEMA_VERSION -from ._utils._project_files import read_toml_project +from ._utils._project_files import read_toml_project, resolve_existing_project_id from .middlewares import Middlewares from .models.runtime_schema import Bindings, EntryPoint from .models.uipath_json_schema import UiPathJsonConfig @@ -44,7 +49,7 @@ console = ConsoleLogger() logger = logging.getLogger(__name__) -CONFIG_PATH = "uipath.json" +CONFIG_PATH = UIPATH_CONFIG_FILE GRAPH_INDENT = " " @@ -54,32 +59,8 @@ class Action(str, enum.Enum): UPDATED = "Updated" -def create_telemetry_config_file(target_directory: str) -> None: - """Create telemetry file if telemetry is enabled. - - Args: - target_directory: The directory where the .uipath folder should be created. - """ - telemetry_enabled = os.getenv(ENV_TELEMETRY_ENABLED, "true").lower() == "true" - - if not telemetry_enabled: - return - - uipath_dir = os.path.join(target_directory, ".uipath") - telemetry_file = os.path.join(uipath_dir, _TELEMETRY_CONFIG_FILE) - - if os.path.exists(telemetry_file): - return - - os.makedirs(uipath_dir, exist_ok=True) - telemetry_data = {_PROJECT_KEY: UiPathConfig.project_id or str(uuid.uuid4())} - - with open(telemetry_file, "w") as f: - json.dump(telemetry_data, f, indent=4) - - def generate_env_file(target_directory): - env_path = os.path.join(target_directory, ".env") + env_path = os.path.join(target_directory, DOTENV_FILE) if not os.path.exists(env_path): relative_path = os.path.relpath(env_path, target_directory) @@ -177,6 +158,27 @@ def write_bindings_file(bindings: Bindings) -> Path: return bindings_file_path +def mark_transaction_root_entrypoints( + entry_point_schemas: list[UiPathRuntimeSchema], +) -> None: + """Stamp entrypoints with 'isTransactionRoot: true' for vertical-solution projects. + + Reads 'runtimeOptions._uipathVerticalSolution' from uipath.json; when true, + marks every entrypoint schema in-place so the flag serializes into + entry-points.json. Otherwise leaves the schemas untouched, so the flag is + dropped on regeneration. + + Args: + entry_point_schemas: The discovered entrypoint schemas. + """ + config = UiPathJsonConfig.load_from_file(str(UiPathConfig.config_file_path)) + if config.runtime_options.uipath_vertical_solution is not True: + return + + for schema in entry_point_schemas: + schema.is_transaction_root = True + + def write_entry_points_file(entry_points: list[UiPathRuntimeSchema]) -> Path: """Write entrypoints to a JSON file. @@ -188,7 +190,7 @@ def write_entry_points_file(entry_points: list[UiPathRuntimeSchema]) -> Path: """ json_object = { "$schema": "https://cloud.uipath.com/draft/2024-12/entry-point", - "$id": "entry-points.json", + "$id": ENTRY_POINTS_FILE, "entryPoints": [ ep.model_dump( by_alias=True, @@ -222,7 +224,9 @@ def write_uiproj_file( ] project_type = determine_project_type(entry_point_models).capitalize() - toml_data = read_toml_project(os.path.join(current_directory, "pyproject.toml")) + toml_data = read_toml_project( + os.path.join(current_directory, PYTHON_CONFIGURATION_FILE) + ) project_name = toml_data["name"] project_description = toml_data.get("description") @@ -277,6 +281,12 @@ def write_studio_metadata_file(directory: str) -> None: ) +MERMAID_FILE_HEADER = ( + "%% AUTO-GENERATED by `uipath init`. Do not edit manually.\n" + "%% Regenerated on every `uipath init`.\n" +) + + def write_mermaid_files(entry_points: list[UiPathRuntimeSchema]) -> list[Path]: """Write mermaid diagram files for each entrypoint. @@ -299,6 +309,7 @@ def write_mermaid_files(entry_points: list[UiPathRuntimeSchema]) -> list[Path]: mermaid_file_path = Path(os.getcwd()) / f"{ep.file_path}.mermaid" with open(mermaid_file_path, "w") as f: + f.write(MERMAID_FILE_HEADER) f.write(str(chart)) mermaid_paths.append(mermaid_file_path) @@ -414,7 +425,6 @@ def init(no_agents_md_override: bool) -> None: with console.spinner("Initializing UiPath project ..."): current_directory = os.getcwd() generate_env_file(current_directory) - create_telemetry_config_file(current_directory) async def initialize() -> list[UiPathRuntimeSchema]: try: @@ -422,10 +432,25 @@ async def initialize() -> list[UiPathRuntimeSchema]: config_path = UiPathConfig.config_file_path if not config_path.exists(): config = UiPathJsonConfig.create_default() + config.id = resolve_existing_project_id(current_directory) or str( + uuid.uuid4() + ) config.save_to_file(config_path) console.success(f"{Action.CREATED.value} '{config_path}' file.") else: - console.info(f"'{config_path}' already exists, skipping.") + # backfill id if not present + async_config_path = anyio.Path(config_path) + raw_config = json.loads(await async_config_path.read_text()) + if not raw_config.get("id"): + raw_config["id"] = resolve_existing_project_id( + current_directory + ) or str(uuid.uuid4()) + await async_config_path.write_text( + json.dumps(raw_config, indent=2) + ) + console.success( + f"{Action.UPDATED.value} '{config_path}' file with 'id'." + ) # Create bindings.json if it doesn't exist bindings_path = UiPathConfig.bindings_file_path @@ -469,6 +494,8 @@ async def initialize() -> list[UiPathRuntimeSchema]: finally: await factory.dispose() + mark_transaction_root_entrypoints(entry_point_schemas) + # Write entry-points.json with all schemas entry_points_path = write_entry_points_file(entry_point_schemas) console.success( diff --git a/packages/uipath/src/uipath/_cli/cli_invoke.py b/packages/uipath/src/uipath/_cli/cli_invoke.py index 726b623d1..62bee75ed 100644 --- a/packages/uipath/src/uipath/_cli/cli_invoke.py +++ b/packages/uipath/src/uipath/_cli/cli_invoke.py @@ -6,6 +6,8 @@ import click import httpx +from uipath.platform.constants import PYTHON_CONFIGURATION_FILE + from .._utils._ssl_context import get_httpx_client_kwargs from ._telemetry import track_command from ._utils._common import get_env_vars @@ -20,7 +22,7 @@ def _read_project_details() -> tuple[str, str]: current_path = os.getcwd() - toml_path = os.path.join(current_path, "pyproject.toml") + toml_path = os.path.join(current_path, PYTHON_CONFIGURATION_FILE) if not os.path.isfile(toml_path): console.error("pyproject.toml not found.") diff --git a/packages/uipath/src/uipath/_cli/cli_list_models.py b/packages/uipath/src/uipath/_cli/cli_list_models.py new file mode 100644 index 000000000..7c14686a4 --- /dev/null +++ b/packages/uipath/src/uipath/_cli/cli_list_models.py @@ -0,0 +1,64 @@ +from collections.abc import Iterable + +import click +from rich.console import Console +from rich.table import Table + +from ..platform.agenthub import LlmModel +from ._utils._context import get_cli_context +from ._utils._service_base import ServiceCommandBase, service_command + + +@click.command(name="list-models") +@click.option( + "--format", + type=click.Choice(["json", "table", "csv"]), + help="Output format (overrides global)", +) +@click.option( + "--output", + "--output-file", + "-o", + type=click.Path(), + help="File path where the output will be written", +) +@service_command +async def list_models(ctx, format, output): + """List available LLM models.""" + client = ServiceCommandBase.get_client(ctx) + models = await client.agenthub.get_available_llm_models_async() + + fmt = format or get_cli_context(ctx).output_format + if fmt == "table" and not output: + _render_rich_table(models) + return None + return models + + +def _render_rich_table(models: Iterable[LlmModel]) -> None: + """Render models as a rich table with one column per vendor.""" + by_vendor: dict[str, list[str]] = {} + for model in models: + vendor = model.vendor or "Unknown" + by_vendor.setdefault(vendor, []).append(model.model_name) + + console = Console() + if not by_vendor: + console.print("Available LLM Models: none") + return + + for names in by_vendor.values(): + names.sort() + + vendors = sorted(by_vendor.keys()) + + table = Table(title="Available LLM Models", show_lines=False) + for vendor in vendors: + table.add_column(vendor, style="cyan", no_wrap=True) + + max_rows = max(len(by_vendor[v]) for v in vendors) + for i in range(max_rows): + row = [by_vendor[v][i] if i < len(by_vendor[v]) else "" for v in vendors] + table.add_row(*row) + + console.print(table) diff --git a/packages/uipath/src/uipath/_cli/cli_new.py b/packages/uipath/src/uipath/_cli/cli_new.py index 390c581c5..0a7723d05 100644 --- a/packages/uipath/src/uipath/_cli/cli_new.py +++ b/packages/uipath/src/uipath/_cli/cli_new.py @@ -4,6 +4,8 @@ import click +from uipath.platform.constants import PYTHON_CONFIGURATION_FILE, UIPATH_CONFIG_FILE + from ._telemetry import track_command from ._utils._console import ConsoleLogger from .middlewares import Middlewares @@ -21,14 +23,14 @@ def generate_script(target_directory): def generate_pyproject(target_directory, project_name): - project_toml_path = os.path.join(target_directory, "pyproject.toml") + project_toml_path = os.path.join(target_directory, PYTHON_CONFIGURATION_FILE) toml_content = f"""[project] name = "{project_name}" version = "0.0.1" description = "{project_name}" authors = [{{ name = "John Doe", email = "john.doe@myemail.com" }}] dependencies = [ - "uipath>=2.2.0, <2.3.0" + "uipath>=2.10.0, <2.11.0" ] requires-python = ">=3.11" """ @@ -38,7 +40,7 @@ def generate_pyproject(target_directory, project_name): def generate_uipath_json(target_directory): - uipath_json_path = os.path.join(target_directory, "uipath.json") + uipath_json_path = os.path.join(target_directory, UIPATH_CONFIG_FILE) uipath_config = {"functions": {"main": "main.py:main"}} with open(uipath_json_path, "w") as f: @@ -74,9 +76,9 @@ def new(name: str): generate_script(directory) console.success("Created 'main.py' file.") generate_pyproject(directory, name) - console.success("Created 'pyproject.toml' file.") + console.success(f"Created '{PYTHON_CONFIGURATION_FILE}' file.") generate_uipath_json(directory) - console.success("Created 'uipath.json' file.") + console.success(f"Created '{UIPATH_CONFIG_FILE}' file.") init_command = """uipath init""" run_command = """uipath run main '{"message": "Hello World!"}'""" console.hint(f""" Initialize project: {click.style(init_command, fg="cyan")}""") diff --git a/packages/uipath/src/uipath/_cli/cli_pack.py b/packages/uipath/src/uipath/_cli/cli_pack.py index 83cedf870..510eeadff 100644 --- a/packages/uipath/src/uipath/_cli/cli_pack.py +++ b/packages/uipath/src/uipath/_cli/cli_pack.py @@ -8,19 +8,26 @@ from pydantic import TypeAdapter from uipath._cli.models.runtime_schema import Bindings, EntryPoint, EntryPoints -from uipath._cli.models.uipath_json_schema import RuntimeOptions, UiPathJsonConfig +from uipath._cli.models.uipath_json_schema import UiPathJsonConfig from uipath.eval.constants import EVALS_FOLDER, LEGACY_EVAL_FOLDER from uipath.platform.common import UiPathConfig +from uipath.platform.constants import ( + ENTRY_POINTS_FILE, + PYTHON_CONFIGURATION_FILE, + UIPATH_BINDINGS_FILE, + UIPATH_CONFIG_FILE, +) -from ..telemetry._constants import _PROJECT_KEY, _TELEMETRY_CONFIG_FILE from ._telemetry import track_command from ._utils._common import determine_project_type from ._utils._console import ConsoleLogger from ._utils._project_files import ( + FileInfo, ensure_config_file, files_to_include, get_project_config, read_toml_project, + resolve_existing_project_id, validate_config, ) from ._utils._uv_helpers import handle_uv_operations @@ -29,34 +36,15 @@ schema = "https://cloud.uipath.com/draft/2024-12/entry-point" +pack_options_spec_url = "https://github.com/UiPath/uipath-python/blob/main/packages/uipath/specs/uipath.spec.md#4-packoptions" -def get_project_id() -> str: - """Get project ID from telemetry file if it exists, otherwise generate a new one. - - Returns: - Project ID string (either from telemetry file or newly generated). - """ - # first check if this is a studio project - if project_id := UiPathConfig.project_id: - return project_id - - telemetry_file = os.path.join(".uipath", _TELEMETRY_CONFIG_FILE) - - if os.path.exists(telemetry_file): - try: - with open(telemetry_file, "r") as f: - telemetry_data = json.load(f) - project_id = telemetry_data.get(_PROJECT_KEY) - if project_id: - return project_id - except (json.JSONDecodeError, IOError): - pass - return str(uuid.uuid4()) +class PackageMetadataConflictError(Exception): + """Raised when project files would be packaged over generated package metadata.""" def get_project_version(directory): - toml_path = os.path.join(directory, "pyproject.toml") + toml_path = os.path.join(directory, PYTHON_CONFIGURATION_FILE) if not os.path.exists(toml_path): console.warning("pyproject.toml not found. Using default version 0.0.1") return "0.0.1" @@ -72,14 +60,27 @@ def validate_config_structure(config_data): def generate_operate_file( - entrypoints: list[EntryPoint], runtimeOptions: RuntimeOptions, dependencies=None + entrypoints: list[EntryPoint], + config: UiPathJsonConfig, + dependencies=None, + directory: str = ".", ): if not entrypoints: raise ValueError( "No entry points found in entry-points.json. Please run 'uipath init' to generate valid entry points." ) - project_id = get_project_id() + # prefer id from uipath.json; fall back to the legacy + # .telemetry.json or SW project id. + if config.id: + try: + uuid.UUID(config.id) + except ValueError: + console.error(f"uipath.json 'id' must be a valid GUID, got '{config.id}'.") + + project_id = ( + config.id or resolve_existing_project_id(directory) or str(uuid.uuid4()) + ) project_type = determine_project_type(entrypoints) first_entry = entrypoints[0] @@ -94,7 +95,7 @@ def generate_operate_file( "runtimeOptions": { "requiresUserInteraction": False, "isAttended": False, - "isConversational": runtimeOptions.is_conversational, + "isConversational": config.runtime_options.is_conversational, }, } @@ -107,7 +108,7 @@ def generate_operate_file( def generate_entrypoints_file(entrypoints: list[EntryPoint]): entrypoint_json_data = { "$schema": schema, - "$id": "entry-points.json", + "$id": ENTRY_POINTS_FILE, "entryPoints": [ ep.model_dump(by_alias=True, exclude_none=True) for ep in entrypoints ], @@ -188,8 +189,8 @@ def generate_psmdcp_content(projectName, version, description, authors): def generate_package_descriptor_content(entrypoints: list[EntryPoint]): files = { "operate.json": "content/operate.json", - "entry-points.json": "content/entry-points.json", - "bindings.json": "content/bindings_v2.json", + ENTRY_POINTS_FILE: "content/entry-points.json", + UIPATH_BINDINGS_FILE: "content/bindings_v2.json", } for entry in entrypoints: @@ -211,6 +212,44 @@ def is_venv_dir(d): ) +def archive_path_for(file: FileInfo) -> str: + """Return the path a project file is packaged under.""" + return f"content/{file.relative_path}" + + +def raise_on_metadata_conflicts( + metadata_files: dict[str, str], files: list[FileInfo] +) -> None: + """Reject project files that would be written over generated package metadata. + + The zip format allows several entries to share a name, so a project file + packaged at the same archive path as a generated metadata file yields a + package with duplicate entries that fails at extraction time. + + Args: + metadata_files: Archive path -> content of the generated metadata files + files: Project files that would be packaged + + Raises: + PackageMetadataConflictError: If any project file collides with metadata + """ + reserved = {path.casefold() for path in metadata_files} + conflicts = sorted( + file.relative_path + for file in files + if archive_path_for(file).casefold() in reserved + ) + if not conflicts: + return + + conflict_list = "\n".join(f" - {path}" for path in conflicts) + raise PackageMetadataConflictError( + f"These project files clash with generated package metadata:\n{conflict_list}\n" + "Delete, rename, or exclude them via packOptions.filesExcluded: " + f"{pack_options_spec_url}" + ) + + def pack_fn( project_name, description, @@ -224,14 +263,16 @@ def pack_fn( directory, str(UiPathConfig.entry_points_file_path) ) if not os.path.exists(entry_points_file_path): - raise Exception("'entry-points.json' file not found. Please run 'uipath init'.") + raise Exception( + f"'{ENTRY_POINTS_FILE}' file not found. Please run 'uipath init'." + ) with open(entry_points_file_path, "r") as f: entry_points_data = EntryPoints.model_validate(json.load(f)) entrypoints = entry_points_data.entrypoints - config_path = os.path.join(directory, "uipath.json") + config_path = os.path.join(directory, UIPATH_CONFIG_FILE) if not os.path.exists(config_path): console.error("uipath.json not found, please run `uipath init`.") @@ -239,10 +280,11 @@ def pack_fn( config_data = TypeAdapter(UiPathJsonConfig).validate_python(json.load(f)) operate_file = generate_operate_file( - entrypoints, config_data.runtime_options, dependencies + entrypoints, config_data, dependencies, directory ) # try to read bindings from bindings.json + bindings_data: Bindings | None = None bindings_path = os.path.join(directory, str(UiPathConfig.bindings_file_path)) if os.path.exists(bindings_path): with open(bindings_path, "r") as f: @@ -261,57 +303,59 @@ def pack_fn( ) package_descriptor_content = generate_package_descriptor_content(entrypoints) + metadata_files = { + f"./package/services/metadata/core-properties/{psmdcp_file_name}": psmdcp_content, + "[Content_Types].xml": content_types_content, + "content/package-descriptor.json": json.dumps( + package_descriptor_content, indent=4 + ), + "content/operate.json": json.dumps(operate_file, indent=4), + } + if bindings_data: + metadata_files["content/bindings_v2.json"] = json.dumps( + bindings_data.model_dump(by_alias=True), indent=4 + ) + metadata_files[f"{project_name}.nuspec"] = nuspec_content + metadata_files["_rels/.rels"] = rels_content + + files, skipped_files = files_to_include( + config_data.pack_options, + directory, + include_uv_lock, + directories_to_ignore=[LEGACY_EVAL_FOLDER, EVALS_FOLDER], + ) + + raise_on_metadata_conflicts(metadata_files, files) + # Create .uipath directory if it doesn't exist os.makedirs(".uipath", exist_ok=True) with zipfile.ZipFile( f".uipath/{project_name}.{version}.nupkg", "w", zipfile.ZIP_DEFLATED ) as z: - # Add metadata files - z.writestr( - f"./package/services/metadata/core-properties/{psmdcp_file_name}", - psmdcp_content, - ) - z.writestr("[Content_Types].xml", content_types_content) - z.writestr( - "content/package-descriptor.json", - json.dumps(package_descriptor_content, indent=4), - ) - z.writestr("content/operate.json", json.dumps(operate_file, indent=4)) - if bindings_data: - z.writestr( - "content/bindings_v2.json", - json.dumps(bindings_data.model_dump(by_alias=True), indent=4), - ) - z.writestr(f"{project_name}.nuspec", nuspec_content) - z.writestr("_rels/.rels", rels_content) - - files, skipped_files = files_to_include( - config_data.pack_options, - directory, - include_uv_lock, - directories_to_ignore=[LEGACY_EVAL_FOLDER, EVALS_FOLDER], - ) + for archive_path, content in metadata_files.items(): + z.writestr(archive_path, content) for file in files: + archive_path = archive_path_for(file) if file.is_binary: # Read binary files in binary mode with open(file.file_path, "rb") as f: - z.writestr(f"content/{file.relative_path}", f.read()) + z.writestr(archive_path, f.read()) else: try: # Try UTF-8 first with open(file.file_path, "r", encoding="utf-8") as f: - z.writestr(f"content/{file.relative_path}", f.read()) + z.writestr(archive_path, f.read()) except UnicodeDecodeError: # If UTF-8 fails, try with utf-8-sig (for files with BOM) try: with open(file.file_path, "r", encoding="utf-8-sig") as f: - z.writestr(f"content/{file.relative_path}", f.read()) + z.writestr(archive_path, f.read()) except UnicodeDecodeError: # If that also fails, try with latin-1 as a fallback with open(file.file_path, "r", encoding="latin-1") as f: - z.writestr(f"content/{file.relative_path}", f.read()) + z.writestr(archive_path, f.read()) def display_project_info(config): @@ -366,6 +410,8 @@ def pack(root, nolock): display_project_info(config) console.success("Project successfully packaged.") + except PackageMetadataConflictError as e: + console.error(str(e)) except Exception as e: console.error( f"Failed to create package {config['project_name']}.{version or config['version']}: {str(e)}" diff --git a/packages/uipath/src/uipath/_cli/cli_push.py b/packages/uipath/src/uipath/_cli/cli_push.py index 61e46cbc6..9082d747f 100644 --- a/packages/uipath/src/uipath/_cli/cli_push.py +++ b/packages/uipath/src/uipath/_cli/cli_push.py @@ -5,9 +5,17 @@ import click from uipath.platform.common import UiPathConfig -from uipath.platform.errors import EnrichedException, FolderNotFoundException - -from ..platform.resource_catalog import ResourceType +from uipath.platform.errors import EnrichedException + +from ._push._resolvers import resolve_bindings +from ._push._resource_actions import ( + CreateReference, + CreateVirtual, + ResourceAction, + Skip, +) +from ._push._summary import ResourceImportSummary +from ._push._virtual_kinds import fetch_supported_virtual_kinds from ._push.sw_file_handler import SwFileHandler from ._telemetry import track_command from ._utils._common import ensure_coded_agent_project, may_override_files @@ -22,10 +30,9 @@ ) from ._utils._studio_project import ( ProjectLockUnavailableError, - ReferencedResourceFolder, - ReferencedResourceRequest, Status, StudioClient, + VirtualResourceRequest, ) from ._utils._uv_helpers import handle_uv_operations from .models.runtime_schema import Bindings @@ -42,133 +49,104 @@ def get_org_scoped_url(base_url: str) -> str: return org_scoped_url -async def create_resources(studio_client: StudioClient): +async def create_resources(studio_client: StudioClient) -> None: console.info("\nImporting referenced resources to Studio Web project...") from uipath.platform import UiPath uipath = UiPath() - resource_catalog = uipath.resource_catalog - connections = uipath.connections with open(UiPathConfig.bindings_file_path, "r") as f: - bindings_file_content = f.read() - - bindings = Bindings.model_validate_json(bindings_file_content) - - resources_not_found = 0 - resources_unchanged = 0 - resources_created = 0 - resource_updated = 0 - - for bindings_resource in bindings.resources: - not_found_warning = "was not found and will not be added to the solution." - found_resource = None - resource_type = bindings_resource.resource - if resource_type == "connection": - connection_key_resource_value = bindings_resource.value.get("ConnectionId") - assert connection_key_resource_value - connection_key = connection_key_resource_value.default_value - try: - connection = await connections.retrieve_async(connection_key) - except EnrichedException: - resources_not_found += 1 - assert bindings_resource.metadata is not None - connector_name = bindings_resource.metadata.get("Connector") - console.warning( - f"Connection with key '{connection_key}' of type '{connector_name}' " - f"{not_found_warning}" - ) - continue - resource_name = connection.name - folder_path = connection.folder.get("path") - else: - name_resource_value = bindings_resource.value.get("name") - folder_path_resource_value = bindings_resource.value.get("folderPath") - - if not folder_path_resource_value: - # guardrail resource, nothing to import - continue - - assert name_resource_value - resource_name = name_resource_value.default_value - folder_path = folder_path_resource_value.default_value - - resources = resource_catalog.list_by_type_async( - resource_type=ResourceType.from_string(resource_type), - name=resource_name, - folder_path=folder_path, - ) + bindings = Bindings.model_validate_json(f.read()) - try: - async for resource in resources: - found_resource = resource - break - await resources.aclose() + supported_virtual_kinds = await fetch_supported_virtual_kinds(studio_client) - except FolderNotFoundException: - pass + summary = ResourceImportSummary() + async for action in resolve_bindings( + bindings, + uipath.resource_catalog, + uipath.connections, + supported_virtual_kinds, + ): + await _execute_action(action, studio_client, summary) - if not found_resource: - console.warning( - f"Resource '{resource_name}' of type '{resource_type}' at folder path '{folder_path}' " - f"{not_found_warning}" - ) - resources_not_found += 1 - continue - - referenced_resource_request = ReferencedResourceRequest( - key=found_resource.resource_key, - kind=found_resource.resource_type, - type=found_resource.resource_sub_type, - folder=next( - ReferencedResourceFolder( - folder_key=folder.key, - fully_qualified_name=folder.fully_qualified_name, - path=folder.path, - ) - for folder in found_resource.folders - ), - ) - response = await studio_client.create_referenced_resource( - referenced_resource_request - ) + console.info(str(summary)) - resource_details = ( - f"(kind = {click.style(found_resource.resource_type, fg='cyan')}, " - f"type = {click.style(found_resource.resource_sub_type, fg='cyan')})" - ) - match response.status: - case Status.ADDED: - console.success( - f"Created reference for resource: {click.style(resource_name, fg='cyan')} " - f"{resource_details}" - ) - resources_created += 1 - case Status.UNCHANGED: - console.info( - f"Resource reference already exists ({click.style('unchanged', fg='yellow')}): {click.style(resource_name, fg='cyan')} " - f"{resource_details}" - ) - resources_unchanged += 1 - case Status.UPDATED: - console.info( - f"Resource reference already exists ({click.style('updated', fg='blue')}): {click.style(resource_name, fg='cyan')} " - f"{resource_details}" - ) - resource_updated += 1 +async def _execute_action( + action: ResourceAction, + studio_client: StudioClient, + summary: ResourceImportSummary, +) -> None: + match action: + case Skip(message=message): + console.warning(message) + summary.not_found += 1 - total_resources = ( - resources_created + resources_unchanged + resources_not_found + resource_updated - ) - console.info( - f"\n \U0001f535 Resource import summary: {total_resources} total resources - " - f"{click.style(str(resources_created), fg='green')} created, " - f"{click.style(str(resource_updated), fg='blue')} updated, " - f"{click.style(str(resources_unchanged), fg='yellow')} unchanged, " - f"{click.style(str(resources_not_found), fg='red')} not found" - ) + case CreateVirtual(request=request): + try: + result = await studio_client.create_virtual_resource(request) + except EnrichedException as e: + console.warning( + f"Failed to create virtual resource '{request.name}' of type " + f"'{request.kind}': {e}" + ) + summary.not_found += 1 + return + label = _format_virtual_label(request) + match result.status: + case Status.ADDED: + console.success(f"{label} created successfully.") + summary.virtual_created += 1 + case Status.UNCHANGED: + console.info(f"{label} already exists. Skipping...") + summary.virtual_existing += 1 + + case CreateReference( + request=request, + resource_name=resource_name, + kind=kind, + sub_type=sub_type, + ): + response = await studio_client.create_referenced_resource(request) + details = ( + f"(kind = {click.style(kind, fg='cyan')}, " + f"type = {click.style(sub_type, fg='cyan')})" + ) + match response.status: + case Status.ADDED: + console.success( + f"Created reference for resource: " + f"{click.style(resource_name, fg='cyan')} {details}" + ) + summary.created += 1 + case Status.UNCHANGED: + console.info( + f"Resource reference already exists " + f"({click.style('unchanged', fg='yellow')}): " + f"{click.style(resource_name, fg='cyan')} {details}" + ) + summary.unchanged += 1 + case Status.UPDATED: + console.info( + f"Resource reference already exists " + f"({click.style('updated', fg='blue')}): " + f"{click.style(resource_name, fg='cyan')} {details}" + ) + summary.updated += 1 + + +def _format_virtual_label(request: VirtualResourceRequest) -> str: + parts = [ + f"Resource {click.style(request.name, fg='cyan')}", + f" (kind: {click.style(request.kind, fg='yellow')}", + ] + if request.type: + parts.append(f", type: {click.style(request.type, fg='yellow')}") + if request.activity_name: + parts.append(f", activity: {click.style(request.activity_name, fg='yellow')}") + parts.append(")") + return "".join(parts) async def upload_source_files_to_project( @@ -262,7 +240,6 @@ def push(root: str, ignore_resources: bool, nolock: bool, overwrite: bool) -> No project_id = UiPathConfig.project_id if not project_id: console.error("UIPATH_PROJECT_ID environment variable not found.") - return studio_client = StudioClient(project_id=project_id) diff --git a/packages/uipath/src/uipath/_cli/cli_run.py b/packages/uipath/src/uipath/_cli/cli_run.py index 6b5152ed4..9d12a86c3 100644 --- a/packages/uipath/src/uipath/_cli/cli_run.py +++ b/packages/uipath/src/uipath/_cli/cli_run.py @@ -1,13 +1,20 @@ import asyncio +from typing import Any import click +from pydantic import ValidationError from uipath._cli._chat._bridge import get_chat_bridge from uipath._cli._debug._bridge import ConsoleDebugBridge from uipath._cli._utils._common import read_resource_overwrites_from_file from uipath._cli._utils._debug import setup_debugging -from uipath.core.tracing import UiPathTraceManager -from uipath.platform.common import ResourceOverwritesContext, UiPathConfig +from uipath._cli._utils._tracing import create_trace_manager +from uipath.eval.mocks import SimulationConfig, UiPathMockRuntime, build_mocking_context +from uipath.platform.common import ( + ExecutionSourceContext, + ResourceOverwritesContext, + UiPathConfig, +) from uipath.runtime import ( UiPathExecuteOptions, UiPathRuntimeFactoryProtocol, @@ -27,6 +34,8 @@ LlmOpsHttpExporter, ) +from ._errors import EntrypointDiscoveryException +from ._governance_bootstrap import GovernanceBootstrap, resolve_governance from ._telemetry import track_command from ._utils._console import ConsoleLogger from .middlewares import Middlewares @@ -34,6 +43,21 @@ console = ConsoleLogger() +class _RunDiscoveryError(EntrypointDiscoveryException): + """Raised when entrypoint auto-discovery fails.""" + + def get_usage_help(self) -> list[str]: + lines = super().get_usage_help() + lines.append("") + lines.append( + "Usage: uipath run " + " [-f ]" + ) + if self.entrypoints: + lines.append(f"Example: uipath run {self.entrypoints[0]}") + return lines + + @click.command() @click.argument("entrypoint", required=False) @click.argument("input", required=False, default=None) @@ -85,6 +109,12 @@ is_flag=True, help="Keep the temporary state file even when not resuming and no job id is provided", ) +@click.option( + "--simulation", + required=False, + default=None, + help="Simulation config as a JSON object (same schema as simulation.json)", +) @track_command("run") def run( entrypoint: str | None, @@ -98,6 +128,7 @@ def run( debug: bool, debug_port: int, keep_state_file: bool, + simulation: str | None, ) -> None: """Execute the project.""" input_file = file or input_file @@ -106,6 +137,14 @@ def run( if not setup_debugging(debug, debug_port): console.error(f"Failed to start debug server on port {debug_port}") + simulation_config: SimulationConfig | None = None + if simulation: + try: + simulation_config = SimulationConfig.model_validate_json(simulation) + except (ValidationError, ValueError) as e: + console.error(f"Invalid --simulation config: {e}") + return + result = Middlewares.next( "run", entrypoint, @@ -125,11 +164,6 @@ def run( return if result.should_continue: - if not entrypoint: - console.error("""No entrypoint specified. Please provide the path to the Python function. - Usage: `uipath run [-f ]`""") - return - try: async def execute_runtime( @@ -158,7 +192,7 @@ async def debug_runtime( return ctx.result async def execute() -> None: - trace_manager = UiPathTraceManager() + trace_manager = create_trace_manager() ctx = UiPathRuntimeContext.with_defaults( entrypoint=entrypoint, @@ -181,22 +215,83 @@ async def execute() -> None: async with ResourceOverwritesContext( lambda: read_resource_overwrites_from_file(ctx.runtime_dir) ): - with ctx: + with ExecutionSourceContext(ctx.execution_source), ctx: + base_runtime: UiPathRuntimeProtocol | None = None runtime: UiPathRuntimeProtocol | None = None chat_runtime: UiPathRuntimeProtocol | None = None factory: UiPathRuntimeFactoryProtocol | None = None + governance_bootstrap: GovernanceBootstrap | None = None try: factory = UiPathRuntimeFactoryRegistry.get(context=ctx) + + resolved_entrypoint = entrypoint + if not resolved_entrypoint: + available = factory.discover_entrypoints() + if len(available) == 1: + resolved_entrypoint = available[0] + else: + raise _RunDiscoveryError(available) + factory_settings = await factory.get_settings() trace_settings = ( factory_settings.trace_settings if factory_settings else None ) - runtime = await factory.new_runtime( - entrypoint, - ctx.conversation_id or ctx.job_id or "default", + agent_type = ( + factory_settings.agent_type + if factory_settings + else None ) + agent_framework = ( + factory_settings.agent_framework + if factory_settings + else None + ) + governance_bootstrap = await resolve_governance( + agent_framework=agent_framework, + agent_type=agent_type, + is_conversational=ctx.conversation_id is not None, + ) + governance_runtime_id = ( + ctx.conversation_id or ctx.job_id or "default" + ) + new_runtime_kwargs: dict[str, Any] = {} + if governance_bootstrap is not None: + new_runtime_kwargs["evaluator"] = ( + governance_bootstrap.evaluator + ) + + base_runtime = await factory.new_runtime( + resolved_entrypoint, + governance_runtime_id, + **new_runtime_kwargs, + ) + + if governance_bootstrap is not None: + base_runtime = governance_bootstrap.wrap_runtime( + base_runtime, + agent_name=resolved_entrypoint, + runtime_id=governance_runtime_id, + ) + + runtime = base_runtime + + if simulation_config: + schema = await base_runtime.get_schema() + agent_model = None + if schema.metadata and "settings" in schema.metadata: + agent_model = schema.metadata["settings"].get( + "model" + ) + mocking_context = build_mocking_context( + simulation_config, agent_model + ) + if mocking_context: + runtime = UiPathMockRuntime( + delegate=base_runtime, + mocking_context=mocking_context, + ) if ctx.job_id: if UiPathConfig.is_tracing_enabled: @@ -221,15 +316,31 @@ async def execute() -> None: else: ctx.result = await debug_runtime(ctx, runtime) finally: - if chat_runtime: - await chat_runtime.dispose() - if runtime: - await runtime.dispose() - if factory: - await factory.dispose() + try: + if chat_runtime: + await chat_runtime.dispose() + if runtime is not None and runtime is not base_runtime: + await runtime.dispose() + if base_runtime is not None: + await base_runtime.dispose() + if governance_bootstrap is not None: + governance_bootstrap.dispose() + if factory: + await factory.dispose() + finally: + trace_manager.shutdown() asyncio.run(execute()) + except _RunDiscoveryError as e: + click.echo("\n".join(e.get_usage_help())) + if not e.entrypoints: + click.echo() + console.link( + "uipath.json spec:", + "https://github.com/UiPath/uipath-python/blob/main/packages/uipath/specs/uipath.spec.md", + ) + return except UiPathRuntimeError as e: console.error(f"{e.error_info.title} - {e.error_info.detail}") except Exception as e: diff --git a/packages/uipath/src/uipath/_cli/cli_server.py b/packages/uipath/src/uipath/_cli/cli_server.py index c32c6de11..d822bdd8c 100644 --- a/packages/uipath/src/uipath/_cli/cli_server.py +++ b/packages/uipath/src/uipath/_cli/cli_server.py @@ -2,7 +2,6 @@ import importlib import json import os -import shlex import sys import tempfile import time @@ -13,44 +12,41 @@ import click from aiohttp import ClientSession, UnixConnector, web +from ._server_core import ( + COMMANDS, + _run_command_isolated, + _state, + parse_args, +) from ._telemetry import track_command from ._utils._console import ConsoleLogger -from .cli_debug import debug -from .cli_eval import eval -from .cli_run import run +from .cli_server_ipc import ( + IPythonRuntimeServer, + PythonRuntimeService, + RunJobRequest, + RunJobResult, + StopJobRequest, + start_ipc_server, +) + +__all__ = [ + "server", + "IPythonRuntimeServer", + "PythonRuntimeService", + "RunJobRequest", + "RunJobResult", + "StopJobRequest", + "start_ipc_server", +] console = ConsoleLogger() +IS_WINDOWS = sys.platform == "win32" + SOCKET_ENV_VAR = "UIPATH_SERVER_SOCKET" DEFAULT_SOCKET_PATH = "/tmp/uipath-server.sock" DEFAULT_PORT = 8765 -IS_WINDOWS = sys.platform == "win32" - -COMMANDS = { - "run": run, - "debug": debug, - "eval": eval, -} - - -class _ServerState: - """Mutable server state, initialized lazily at server startup.""" - - def __init__(self) -> None: - self.lock: asyncio.Lock | None = None - self.baseline_env: dict[str, str] | None = None - - def init(self) -> None: - """Must be called inside a running event loop at server startup.""" - if self.lock is not None: - return - self.lock = asyncio.Lock() - self.baseline_env = os.environ.copy() - - -_state = _ServerState() - DEFAULT_PRELOAD_MODULES = [ # Network/async - slowest to load @@ -83,9 +79,10 @@ def preload_modules() -> None: for module_name in modules_to_load: if module_name in sys.modules: continue - if find_spec(module_name) is None: - continue try: + # find_spec raises ModuleNotFoundError when a parent package is missing + if find_spec(module_name) is None: + continue importlib.import_module(module_name) console.success(f"Pre-loaded module: {module_name}") except ImportError as e: @@ -96,7 +93,7 @@ def preload_modules() -> None: def generate_socket_path() -> str: - """Generate a unique socket path for the server to listen on.""" + """Generate a unique socket path for the HTTP server to listen on.""" return os.path.join(tempfile.gettempdir(), f"uipath-server-{os.getpid()}.sock") @@ -108,15 +105,9 @@ def get_field(message: dict[str, Any], *keys: str) -> Any: return None -def parse_args(args: str | list[str] | None) -> list[str]: - """Parse args into a list of strings.""" - if args is None: - return [] - if isinstance(args, list): - return args - if isinstance(args, str): - return shlex.split(args) - return [] +# --------------------------------------------------------------------------- # +# HTTP transport (default) — aiohttp over a Unix socket / TCP, with ready-ACK # +# --------------------------------------------------------------------------- # async def send_ack(ack_socket_path: str, server_socket_path: str) -> None: @@ -149,7 +140,7 @@ async def handle_health(request: web.Request) -> web.Response: async def handle_start(request: web.Request) -> web.Response: - """Handle POST /jobs/{job_key}/start endpoint.""" + """Handle POST /jobs/{job_key}/start — runs a job via the shared core.""" job_key = request.match_info.get("job_key") if not job_key: return web.json_response( @@ -172,13 +163,19 @@ async def handle_start(request: web.Request) -> web.Response: status=400, ) - args_raw = get_field(message, "args", "Args") - args = parse_args(args_raw) - - env_vars = get_field(message, "environmentVariables", "EnvironmentVariables") or {} + args = parse_args(get_field(message, "args", "Args")) + env_vars = get_field(message, "environmentVariables", "EnvironmentVariables") working_dir = get_field(message, "workingDirectory", "WorkingDirectory") - console.info(f"Starting job {job_key}: {command_name} {args}") + if env_vars is not None and not isinstance(env_vars, dict): + return web.json_response( + { + "success": False, + "error": "Invalid field: 'environmentVariables' must be a dict", + }, + status=400, + ) + env_vars = env_vars or {} cmd = COMMANDS.get(command_name) if cmd is None: @@ -187,78 +184,28 @@ async def handle_start(request: web.Request) -> web.Response: status=400, ) - console.info(f"Original cwd: {os.getcwd()}") - console.info(f"Requested working_dir: {working_dir}") + console.info(f"Starting job {job_key}: {command_name} {args}") - if _state.lock is None or _state.baseline_env is None: - raise RuntimeError("Server state not initialized") + result = await _run_command_isolated(cmd, args, env_vars, working_dir) - # Validate environmentVariables type early - if env_vars and not isinstance(env_vars, dict): + if result["Unexpected"]: return web.json_response( - { - "success": False, - "error": "Invalid field: 'environmentVariables' must be a dict", - }, + {"success": False, "job_key": job_key, "error": result["Error"]}, + status=500, + ) + if result.get("ClientError"): + # Request-shaped failure (e.g. bad working directory) — 4xx, not 200. + return web.json_response( + {"success": False, "job_key": job_key, "error": result["Error"]}, status=400, ) - - # Serialize command execution to prevent concurrent os.environ mutation - async with _state.lock: - original_cwd = os.getcwd() - - try: - # Start from server baseline + request env vars only. - # This ensures no env vars from previous requests leak through. - os.environ.clear() - os.environ.update(_state.baseline_env) - if isinstance(env_vars, dict): - os.environ.update(env_vars) - - if working_dir and isinstance(working_dir, str): - try: - os.chdir(working_dir) - except (FileNotFoundError, NotADirectoryError, PermissionError) as e: - return web.json_response( - { - "success": False, - "job_key": job_key, - "error": f"Cannot change to working directory: {e}", - }, - status=400, - ) - - result = await asyncio.to_thread(cmd.main, args, standalone_mode=False) - - return web.json_response( - { - "success": True, - "job_key": job_key, - "result": result, - } - ) - except SystemExit as e: - exit_code = e.code if isinstance(e.code, int) else 1 - return web.json_response( - { - "success": exit_code == 0, - "job_key": job_key, - "error": None if exit_code == 0 else f"Exit code: {exit_code}", - } - ) - except Exception as e: - return web.json_response( - {"success": False, "job_key": job_key, "error": str(e)}, - status=500, - ) - finally: - # Restore to server baseline - try: - os.chdir(original_cwd) - except OSError: - pass - os.environ.clear() - os.environ.update(_state.baseline_env) + if result["ExitCode"] == 0: + return web.json_response( + {"success": True, "job_key": job_key, "result": result["Result"]} + ) + return web.json_response( + {"success": False, "job_key": job_key, "error": result["Error"]} + ) ALLOWED_HOSTS = {"127.0.0.1", "localhost", "[::1]"} @@ -349,58 +296,106 @@ async def start_tcp_server(host: str, port: int) -> None: await runner.cleanup() +# The uipath-ipc transport (contract, DTOs, service, ``start_ipc_server``) lives +# in ``cli_server_ipc`` and is served alongside HTTP when ``--ipc-pipe`` is given. +# Older servers served HTTP only; the .NET Handler copes. + + +# --------------------------------------------------------------------------- # +# CLI # +# --------------------------------------------------------------------------- # + + @click.command() @click.option( "--client-socket", type=str, default=None, - help=f"Unix socket path to send ready ack to (default: ${SOCKET_ENV_VAR} or {DEFAULT_SOCKET_PATH})", + help=f"Unix socket to send the ready ACK to (default: ${SOCKET_ENV_VAR} " + f"or {DEFAULT_SOCKET_PATH}).", ) @click.option( "--server-socket", type=str, default=None, - help="Unix socket path the server listens on (default: auto-generated in tmp dir)", + help="Unix socket the HTTP server listens on (default: auto-generated in tmp).", +) +@click.option( + "--ipc-pipe", + type=str, + default=None, + help="Named pipe for the uipath-ipc channel. IPC is served only when this is " + "given; omit it for HTTP-only.", ) @click.option( "--port", type=int, default=None, - help=f"TCP port, used on Windows or when --tcp flag is set (default: {DEFAULT_PORT})", + help=f"TCP port, used on Windows or with --tcp (default: {DEFAULT_PORT}).", ) @click.option( "--tcp", is_flag=True, - help="Force TCP mode even on Unix systems", + help="Force TCP mode even on Unix systems.", ) @track_command("server") def server( client_socket: str | None, server_socket: str | None, + ipc_pipe: str | None, port: int | None, tcp: bool, ) -> None: - """Start an HTTP server that forwards commands to run/debug/eval. + """Serve run/debug/eval over HTTP, plus uipath-ipc when --ipc-pipe is given.""" + preload_modules() + _run_server(client_socket, server_socket, ipc_pipe, port, tcp) - Creates its own socket to listen on and sends an ack to --client-socket with: - {"status": "ready", "socket": "/path/to/server.sock"} - Endpoint: POST /jobs/{job_key}/start - Body: {"command": "run", "args": "agent.json '{}'", "environmentVariables": {}, "workingDirectory": "/path"} +async def _serve( + ack_socket_path: str, + server_socket: str | None, + ipc_pipe: str | None, + port: int, + use_tcp: bool, +) -> None: + """Run the HTTP channel, plus the uipath-ipc channel when a pipe name is given.""" + _state.init() + + tasks: list[Any] = [] + if use_tcp: + tasks.append(start_tcp_server("127.0.0.1", port)) + else: + tasks.append(start_unix_server(ack_socket_path, server_socket)) - Endpoint: GET /health - """ - use_tcp = IS_WINDOWS or tcp + # IPC is opt-in and independent of the HTTP socket: it is served only when an + # explicit pipe name is given, which both sides agree on out of band (the .NET + # peer connects to the same name it passed — no derivation from the HTTP socket). + if ipc_pipe: + tasks.append(start_ipc_server(ipc_pipe)) - preload_modules() + await asyncio.gather(*tasks) + +def _run_server( + client_socket: str | None, + server_socket: str | None, + ipc_pipe: str | None, + port: int | None, + tcp: bool, +) -> None: + """Drive ``_serve`` on the right event loop for the platform.""" + use_tcp = IS_WINDOWS or tcp + ack_socket_path = ( + client_socket or os.environ.get(SOCKET_ENV_VAR) or DEFAULT_SOCKET_PATH + ) + coro = _serve( + ack_socket_path, server_socket, ipc_pipe, port or DEFAULT_PORT, use_tcp + ) try: - if use_tcp: - asyncio.run(start_tcp_server("127.0.0.1", port or DEFAULT_PORT)) + if sys.platform == "win32": + with asyncio.Runner(loop_factory=asyncio.ProactorEventLoop) as runner: + runner.run(coro) else: - ack_socket_path = ( - client_socket or os.environ.get(SOCKET_ENV_VAR) or DEFAULT_SOCKET_PATH - ) - asyncio.run(start_unix_server(ack_socket_path, server_socket)) + asyncio.run(coro) except KeyboardInterrupt: console.info("Shutting down") diff --git a/packages/uipath/src/uipath/_cli/cli_server_ipc.py b/packages/uipath/src/uipath/_cli/cli_server_ipc.py new file mode 100644 index 000000000..11c461ae6 --- /dev/null +++ b/packages/uipath/src/uipath/_cli/cli_server_ipc.py @@ -0,0 +1,104 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass, field + +from ._server_core import COMMANDS, _run_command_isolated, _state, parse_args +from ._utils._console import ConsoleLogger + +console = ConsoleLogger() + + +def _run_id(job_key: str, resume_version: int | None) -> str: + return job_key if resume_version is None else f"{job_key}-{resume_version}" + + +@dataclass +class RunJobRequest: + """PascalCase fields match the wire keys.""" + + JobKey: str = "" + ResumeVersion: int | None = None + Command: str = "" + # The peer sends a single string; HTTP callers and tests may pass a + # pre-split list. parse_args accepts both. + Args: str | list[str] | None = None + WorkingDirectory: str | None = None + EnvironmentVariables: dict[str, str] = field(default_factory=dict) + + +@dataclass +class StopJobRequest: + JobKey: str = "" + ResumeVersion: int | None = None + ForceStop: bool = False + + +@dataclass +class RunJobResult: + ExitCode: int = 0 + Error: str | None = None + + +class IPythonRuntimeServer(ABC): + """Contract the job executor calls over uipath-ipc.""" + + @abstractmethod + async def RunJob(self, request: RunJobRequest) -> RunJobResult: + """Run a job → RunJobResult(ExitCode, Error).""" + + @abstractmethod + async def StopJob(self, request: StopJobRequest) -> bool: + """Cancel a running job by key (bool return avoids fire-and-forget).""" + + +class PythonRuntimeService(IPythonRuntimeServer): + """``IPythonRuntimeServer`` implementation backed by run/debug/eval.""" + + async def RunJob(self, request: RunJobRequest) -> RunJobResult: + command_name = request.Command + if not isinstance(command_name, str) or not command_name: + return RunJobResult(ExitCode=1, Error="Missing or invalid field: 'Command'") + + cmd = COMMANDS.get(command_name) + if cmd is None: + return RunJobResult(ExitCode=1, Error=f"Unknown command: {command_name}") + + args = parse_args(request.Args) + + console.info( + f"Running job {_run_id(request.JobKey, request.ResumeVersion)}: {command_name} {args}" + ) + + result = await _run_command_isolated( + cmd, args, request.EnvironmentVariables, request.WorkingDirectory + ) + # IPC contract (RunJobResult) carries only ExitCode + Error. + return RunJobResult(ExitCode=result["ExitCode"], Error=result["Error"]) + + async def StopJob(self, request: StopJobRequest) -> bool: + console.info( + f"StopJob requested for {_run_id(request.JobKey, request.ResumeVersion)} " + f"(force={request.ForceStop}) (no-op)" + ) + return True + + +async def start_ipc_server(pipe_name: str) -> None: + """Serve the Python runtime over a uipath-ipc named pipe until it is closed.""" + try: + from uipath_ipc import IpcServer, NamedPipeServerTransport + except ImportError as e: + raise RuntimeError( + "The uipath-ipc channel was requested (--ipc-pipe) but the 'uipath-ipc' " + "package is not installed in this environment. Install it (pip install " + "'uipath[ipc]') or omit --ipc-pipe to serve HTTP only." + ) from e + + _state.init() + server = IpcServer( + transport=NamedPipeServerTransport(pipe_name), + services={IPythonRuntimeServer: PythonRuntimeService()}, + request_timeout=None, # jobs are long-running; no server-side timeout + ) + console.success(f"IPC server listening on pipe '{pipe_name}'") + async with server: + await server.serve_forever() diff --git a/packages/uipath/src/uipath/_cli/models/runtime_schema.py b/packages/uipath/src/uipath/_cli/models/runtime_schema.py index a32660abf..baa62740d 100644 --- a/packages/uipath/src/uipath/_cli/models/runtime_schema.py +++ b/packages/uipath/src/uipath/_cli/models/runtime_schema.py @@ -2,6 +2,8 @@ from pydantic import BaseModel, ConfigDict, Field +from uipath.platform.constants import ENTRY_POINTS_FILE + class BaseModelWithDefaultConfig(BaseModel): model_config = ConfigDict( @@ -48,7 +50,7 @@ class EntryPoints(BaseModelWithDefaultConfig): default="https://cloud.uipath.com/draft/2024-12/entry-point", alias="$schema", ) - id_: str = Field(default="entry-points.json", alias="$id") + id_: str = Field(default=ENTRY_POINTS_FILE, alias="$id") entrypoints: list[EntryPoint] = Field(..., alias="entryPoints") diff --git a/packages/uipath/src/uipath/_cli/models/uipath_json_schema.py b/packages/uipath/src/uipath/_cli/models/uipath_json_schema.py index f1cd30202..ca5818b86 100644 --- a/packages/uipath/src/uipath/_cli/models/uipath_json_schema.py +++ b/packages/uipath/src/uipath/_cli/models/uipath_json_schema.py @@ -4,6 +4,8 @@ from pydantic import BaseModel, ConfigDict, Field +from uipath.platform.constants import UIPATH_CONFIG_FILE + class BaseModelWithDefaultConfig(BaseModel): model_config = ConfigDict( @@ -24,6 +26,13 @@ class RuntimeOptions(BaseModelWithDefaultConfig): alias="isConversational", description="Enable conversational mode for the runtime", ) + uipath_vertical_solution: bool | None = Field( + default=None, + alias="_uipathVerticalSolution", + description="Marks the project as part of a UiPath vertical solution. " + "When true, 'uipath init' stamps 'isTransactionRoot: true' on every " + "entrypoint in entry-points.json.", + ) class DesignOptions(BaseModelWithDefaultConfig): @@ -68,6 +77,12 @@ class UiPathJsonConfig(BaseModelWithDefaultConfig): alias="$schema", description="Reference to the JSON schema for editor support", ) + id: str | None = Field( + default=None, + description="Stable unique identifier for the agent. Minted once at " + "project creation (by 'uipath init' or Studio Web) and preserved for the " + "lifetime of the project. Used as the package 'projectId' at pack time.", + ) runtime_options: RuntimeOptions = Field( default_factory=RuntimeOptions, alias="runtimeOptions", @@ -120,7 +135,7 @@ def create_default(cls) -> "UiPathJsonConfig": ) @classmethod - def load_from_file(cls, file_path: str = "uipath.json") -> "UiPathJsonConfig": + def load_from_file(cls, file_path: str = UIPATH_CONFIG_FILE) -> "UiPathJsonConfig": """Load configuration from a JSON file.""" import json from pathlib import Path diff --git a/packages/uipath/src/uipath/_cli/services/cli_context_grounding.py b/packages/uipath/src/uipath/_cli/services/cli_context_grounding.py index 075ffe869..1e25def5e 100644 --- a/packages/uipath/src/uipath/_cli/services/cli_context_grounding.py +++ b/packages/uipath/src/uipath/_cli/services/cli_context_grounding.py @@ -460,9 +460,9 @@ def ingest_index( ) @click.option( "--search-mode", - type=click.Choice(["Auto", "Semantic"]), - default="Auto", - help="Search mode (default: Auto)", + type=click.Choice(["Semantic"]), + default="Semantic", + help="Search mode (default: Semantic)", ) @common_service_options @service_command diff --git a/packages/uipath/src/uipath/_resources/CLI_REFERENCE.md b/packages/uipath/src/uipath/_resources/CLI_REFERENCE.md index 98524ddfd..ff4009b22 100644 --- a/packages/uipath/src/uipath/_resources/CLI_REFERENCE.md +++ b/packages/uipath/src/uipath/_resources/CLI_REFERENCE.md @@ -923,7 +923,7 @@ Options: - `--query`: Search query in natural language (default: `Sentinel.UNSET`) - `--limit`: Maximum number of results (default: 10) (default: `10`) - `--threshold`: Minimum similarity threshold (default: 0.0) (default: `0.0`) -- `--search-mode`: Search mode (default: Auto) (default: `Auto`) +- `--search-mode`: Search mode (default: Semantic) (default: `Semantic`) - `--folder-path`: Folder path (e.g., "Shared"). Can also be set via UIPATH_FOLDER_PATH environment variable. (default: `Sentinel.UNSET`) - `--folder-key`: Folder key (UUID) (default: `Sentinel.UNSET`) - `--format`: Output format (overrides global) (default: `Sentinel.UNSET`) diff --git a/packages/uipath/src/uipath/_resources/SDK_REFERENCE.md b/packages/uipath/src/uipath/_resources/SDK_REFERENCE.md index 4af1b60ae..02e9c0676 100644 --- a/packages/uipath/src/uipath/_resources/SDK_REFERENCE.md +++ b/packages/uipath/src/uipath/_resources/SDK_REFERENCE.md @@ -62,12 +62,18 @@ sdk.assets.retrieve(name: str, folder_key: Optional[str]=None, folder_path: Opti # Asynchronously retrieve an asset by its name. sdk.assets.retrieve_async(name: str, folder_key: Optional[str]=None, folder_path: Optional[str]=None) -> uipath.platform.orchestrator.assets.UserAsset | uipath.platform.orchestrator.assets.Asset -# Gets a specified Orchestrator credential. +# Get the decrypted value of a Secret asset. sdk.assets.retrieve_credential(name: str, folder_key: Optional[str]=None, folder_path: Optional[str]=None) -> typing.Optional[str] -# Asynchronously gets a specified Orchestrator credential. +# Asynchronously get the decrypted value of a Secret asset. sdk.assets.retrieve_credential_async(name: str, folder_key: Optional[str]=None, folder_path: Optional[str]=None) -> typing.Optional[str] +# Get the decrypted value of a Secret asset. +sdk.assets.retrieve_secret(name: str, folder_key: Optional[str]=None, folder_path: Optional[str]=None) -> typing.Optional[str] + +# Asynchronously get the decrypted value of a Secret asset. +sdk.assets.retrieve_secret_async(name: str, folder_key: Optional[str]=None, folder_path: Optional[str]=None) -> typing.Optional[str] + # Update an asset's value. sdk.assets.update(robot_asset: uipath.platform.orchestrator.assets.UserAsset, folder_key: Optional[str]=None, folder_path: Optional[str]=None) -> httpx.Response @@ -113,6 +119,19 @@ sdk.attachments.upload_async(name: str, content: str | bytes | None=None, source ``` +### Automation Ops + +Automation Ops service + +```python +# Retrieve the deployed policy. +sdk.automation_ops.get_deployed_policy() -> dict[str, typing.Any] + +# Retrieve the deployed policy (async). +sdk.automation_ops.get_deployed_policy_async() -> dict[str, typing.Any] + +``` + ### Automation Tracker Automation Tracker service @@ -272,10 +291,10 @@ sdk.context_grounding.add_to_index(name: str, blob_file_path: str, content_type: sdk.context_grounding.add_to_index_async(name: str, blob_file_path: str, content_type: Optional[str]=None, content: Union[str, bytes, NoneType]=None, source_path: Optional[str]=None, folder_key: Optional[str]=None, folder_path: Optional[str]=None, ingest_data: bool=True) -> None # Create a new ephemeral context grounding index. -sdk.context_grounding.create_ephemeral_index(usage: uipath.platform.context_grounding.context_grounding_index.ContextGroundingIndex +sdk.context_grounding.create_ephemeral_index(usage: uipath.platform.context_grounding.context_grounding_index.ContextGroundingIndex # Create a new ephemeral context grounding index. -sdk.context_grounding.create_ephemeral_index_async(usage: uipath.platform.context_grounding.context_grounding_index.ContextGroundingIndex +sdk.context_grounding.create_ephemeral_index_async(usage: uipath.platform.context_grounding.context_grounding_index.ContextGroundingIndex # Create a new context grounding index. sdk.context_grounding.create_index(name: str, source: Union[uipath.platform.context_grounding.context_grounding_payloads.BucketSourceConfig, uipath.platform.context_grounding.context_grounding_payloads.GoogleDriveSourceConfig, uipath.platform.context_grounding.context_grounding_payloads.DropboxSourceConfig, uipath.platform.context_grounding.context_grounding_payloads.OneDriveSourceConfig, uipath.platform.context_grounding.context_grounding_payloads.ConfluenceSourceConfig], description: Optional[str]=None, extraction_strategy: Optional[str]=None, embeddings_enabled: Optional[bool]=None, is_encrypted: Optional[bool]=None, folder_key: Optional[str]=None, folder_path: Optional[str]=None) -> uipath.platform.context_grounding.context_grounding_index.ContextGroundingIndex @@ -326,7 +345,7 @@ sdk.context_grounding.list_indexes(folder_key: Optional[str]=None, folder_path: sdk.context_grounding.list_indexes_async(folder_key: Optional[str]=None, folder_path: Optional[str]=None) -> typing.List[uipath.platform.context_grounding.context_grounding_index.ContextGroundingIndex] # Retrieve context grounding index information by its name. -sdk.context_grounding.retrieve(name: str, folder_key: Optional[str]=None, folder_path: Optional[str]=None) -> uipath.platform.context_grounding.context_grounding_index.ContextGroundingIndex +sdk.context_grounding.retrieve(name: str, folder_key: Optional[str]=None, folder_path: Optional[str]=None, include_system_indexes: bool=False) -> uipath.platform.context_grounding.context_grounding_index.ContextGroundingIndex # Retrieve all context grounding indexes across all folders. sdk.context_grounding.retrieve_across_folders(name: Optional[str]=None) -> typing.List[uipath.platform.context_grounding.context_grounding_index.ContextGroundingIndex] @@ -335,7 +354,7 @@ sdk.context_grounding.retrieve_across_folders(name: Optional[str]=None) -> typin sdk.context_grounding.retrieve_across_folders_async(name: Optional[str]=None) -> typing.List[uipath.platform.context_grounding.context_grounding_index.ContextGroundingIndex] # Asynchronously retrieve context grounding index information by its name. -sdk.context_grounding.retrieve_async(name: str, folder_key: Optional[str]=None, folder_path: Optional[str]=None) -> uipath.platform.context_grounding.context_grounding_index.ContextGroundingIndex +sdk.context_grounding.retrieve_async(name: str, folder_key: Optional[str]=None, folder_path: Optional[str]=None, include_system_indexes: bool=False) -> uipath.platform.context_grounding.context_grounding_index.ContextGroundingIndex # Retrieves a Batch Transform task status. sdk.context_grounding.retrieve_batch_transform(id: str, index_name: str | None=None) -> uipath.platform.context_grounding.context_grounding.BatchTransformResponse @@ -386,10 +405,10 @@ sdk.context_grounding.start_deep_rag_ephemeral(name: str, prompt: Annotated[str, sdk.context_grounding.start_deep_rag_ephemeral_async(name: str, prompt: Annotated[str, FieldInfo(annotation=NoneType, required=True, metadata=[MaxLen(max_length=250000)])], glob_pattern: Annotated[str, FieldInfo(annotation=NoneType, required=False, default='*', metadata=[MaxLen(max_length=512)])]="**", citation_mode: uipath.platform.context_grounding.context_grounding.DeepRagCreationResponse # Perform a unified search on a context grounding index. -sdk.context_grounding.unified_search(name: str, query: str, search_mode: uipath.platform.context_grounding.context_grounding.UnifiedQueryResult +sdk.context_grounding.unified_search(name: str, query: str, search_mode: uipath.platform.context_grounding.context_grounding.UnifiedQueryResult # Asynchronously perform a unified search on a context grounding index. -sdk.context_grounding.unified_search_async(name: str, query: str, search_mode: uipath.platform.context_grounding.context_grounding.UnifiedQueryResult +sdk.context_grounding.unified_search_async(name: str, query: str, search_mode: uipath.platform.context_grounding.context_grounding.UnifiedQueryResult ``` @@ -475,17 +494,77 @@ sdk.documents.start_ixp_extraction_validation_async(extraction_response: uipath. Entities service ```python +# Create a new entity with the given schema and return its id. +sdk.entities.create_entity(name: str, fields: List[uipath.platform.entities.entities.EntityCreateFieldOptions], options: Optional[uipath.platform.entities.entities.EntityCreateOptions]=None) -> str + +# Asynchronously create a new entity with the given schema. +sdk.entities.create_entity_async(name: str, fields: List[uipath.platform.entities.entities.EntityCreateFieldOptions], options: Optional[uipath.platform.entities.entities.EntityCreateOptions]=None) -> str + +# Remove the file attached to a File-type field on a record. +sdk.entities.delete_attachment(entity_id: str, record_id: str, field_name: str, expansion_level: Optional[int]=None) -> typing.Dict[str, typing.Any] + +# Asynchronously remove the file attached to a File-type field. +sdk.entities.delete_attachment_async(entity_id: str, record_id: str, field_name: str, expansion_level: Optional[int]=None) -> typing.Dict[str, typing.Any] + +# Delete an entity and all of its records. +sdk.entities.delete_entity(entity_id: str) -> None + +# Asynchronously delete an entity and all of its records. +sdk.entities.delete_entity_async(entity_id: str) -> None + +# Delete a single record by id. +sdk.entities.delete_record(entity_key: str, record_id: str) -> None + +# Asynchronously delete a single record by id. +sdk.entities.delete_record_async(entity_key: str, record_id: str) -> None + # Delete multiple records from an entity in a single batch operation. -sdk.entities.delete_records(entity_key: str, record_ids: List[str]) -> uipath.platform.entities.entities.EntityRecordsBatchResponse +sdk.entities.delete_records(entity_key: str, record_ids: List[str], fail_on_first: Optional[bool]=None) -> uipath.platform.entities.entities.EntityRecordsBatchResponse # Asynchronously delete multiple records from an entity in a single batch operation. -sdk.entities.delete_records_async(entity_key: str, record_ids: List[str]) -> uipath.platform.entities.entities.EntityRecordsBatchResponse +sdk.entities.delete_records_async(entity_key: str, record_ids: List[str], fail_on_first: Optional[bool]=None) -> uipath.platform.entities.entities.EntityRecordsBatchResponse + +# Download a file attached to a record and return its raw bytes. +sdk.entities.download_attachment(entity_id: str, record_id: str, field_name: str) -> bytes + +# Asynchronously download a file attached to a record. +sdk.entities.download_attachment_async(entity_id: str, record_id: str, field_name: str) -> bytes + +# Get the values of a choice set by its ID. +sdk.entities.get_choiceset_values(choiceset_id: str, start: Optional[int]=None, limit: Optional[int]=None) -> typing.List[uipath.platform.entities.entities.ChoiceSetValue] + +# Asynchronously get the values of a choice set by its ID. +sdk.entities.get_choiceset_values_async(choiceset_id: str, start: Optional[int]=None, limit: Optional[int]=None) -> typing.List[uipath.platform.entities.entities.ChoiceSetValue] + +# Fetch a single entity record by its id. +sdk.entities.get_record(entity_key: str, record_id: str, expansion_level: Optional[int]=None) -> uipath.platform.entities.entities.EntityRecord + +# Asynchronously fetch a single entity record by its id. +sdk.entities.get_record_async(entity_key: str, record_id: str, expansion_level: Optional[int]=None) -> uipath.platform.entities.entities.EntityRecord + +# Bulk-import records into an entity from a CSV file. +sdk.entities.import_records(entity_id: str, file: Union[bytes, bytearray, memoryview, NoneType]=None, file_path: Optional[str]=None) -> uipath.platform.entities.entities.EntityImportRecordsResponse + +# Asynchronously bulk-import records into an entity from a CSV file. +sdk.entities.import_records_async(entity_id: str, file: Union[bytes, bytearray, memoryview, NoneType]=None, file_path: Optional[str]=None) -> uipath.platform.entities.entities.EntityImportRecordsResponse + +# Insert a single record into an entity and return the inserted row. +sdk.entities.insert_record(entity_key: str, data: Any, expansion_level: Optional[int]=None) -> uipath.platform.entities.entities.EntityRecord + +# Asynchronously insert a single record into an entity. +sdk.entities.insert_record_async(entity_key: str, data: Any, expansion_level: Optional[int]=None) -> uipath.platform.entities.entities.EntityRecord # Insert multiple records into an entity in a single batch operation. -sdk.entities.insert_records(entity_key: str, records: List[Any], schema: Optional[Type[Any]]=None) -> uipath.platform.entities.entities.EntityRecordsBatchResponse +sdk.entities.insert_records(entity_key: str, records: List[Any], schema: Optional[Type[Any]]=None, expansion_level: Optional[int]=None, fail_on_first: Optional[bool]=None) -> uipath.platform.entities.entities.EntityRecordsBatchResponse # Asynchronously insert multiple records into an entity in a single batch operation. -sdk.entities.insert_records_async(entity_key: str, records: List[Any], schema: Optional[Type[Any]]=None) -> uipath.platform.entities.entities.EntityRecordsBatchResponse +sdk.entities.insert_records_async(entity_key: str, records: List[Any], schema: Optional[Type[Any]]=None, expansion_level: Optional[int]=None, fail_on_first: Optional[bool]=None) -> uipath.platform.entities.entities.EntityRecordsBatchResponse + +# List all choice sets in Data Service. +sdk.entities.list_choicesets() -> typing.List[uipath.platform.entities.entities.Entity] + +# Asynchronously list all choice sets in Data Service. +sdk.entities.list_choicesets_async() -> typing.List[uipath.platform.entities.entities.Entity] # List all entities in Data Service. sdk.entities.list_entities() -> typing.List[uipath.platform.entities.entities.Entity] @@ -494,16 +573,22 @@ sdk.entities.list_entities() -> typing.List[uipath.platform.entities.entities.En sdk.entities.list_entities_async() -> typing.List[uipath.platform.entities.entities.Entity] # List records from an entity with optional pagination and schema validation. -sdk.entities.list_records(entity_key: str, schema: Optional[Type[Any]]=None, start: Optional[int]=None, limit: Optional[int]=None) -> typing.List[uipath.platform.entities.entities.EntityRecord] +sdk.entities.list_records(entity_key: str, schema: Optional[Type[Any]]=None, start: Optional[int]=None, limit: Optional[int]=None, expansion_level: Optional[int]=None, filter: Optional[str]=None, orderby: Optional[str]=None, select: Optional[List[str]]=None, expand: Optional[List[str]]=None) -> uipath.platform.entities.entities.EntityRecordsListResponse # Asynchronously list records from an entity with optional pagination and schema validation. -sdk.entities.list_records_async(entity_key: str, schema: Optional[Type[Any]]=None, start: Optional[int]=None, limit: Optional[int]=None) -> typing.List[uipath.platform.entities.entities.EntityRecord] +sdk.entities.list_records_async(entity_key: str, schema: Optional[Type[Any]]=None, start: Optional[int]=None, limit: Optional[int]=None, expansion_level: Optional[int]=None, filter: Optional[str]=None, orderby: Optional[str]=None, select: Optional[List[str]]=None, expand: Optional[List[str]]=None) -> uipath.platform.entities.entities.EntityRecordsListResponse # Query entity records using a validated SQL query. -sdk.entities.query_entity_records(sql_query: str, routing_context: Optional[uipath.platform.entities.entities.QueryRoutingOverrideContext]=None) -> typing.List[typing.Dict[str, typing.Any]] +sdk.entities.query_entity_records(sql_query: str) -> typing.List[typing.Dict[str, typing.Any]] # Asynchronously query entity records using a validated SQL query. -sdk.entities.query_entity_records_async(sql_query: str, routing_context: Optional[uipath.platform.entities.entities.QueryRoutingOverrideContext]=None) -> typing.List[typing.Dict[str, typing.Any]] +sdk.entities.query_entity_records_async(sql_query: str) -> typing.List[typing.Dict[str, typing.Any]] + +# Resolve an agent entity set, applying resource overwrites. +sdk.entities.resolve_entity_set(items: List[uipath.platform.entities.entities.DataFabricEntityItem]) -> uipath.platform.entities.entities.EntitySetResolution + +# Resolve an agent entity set, applying resource overwrites. +sdk.entities.resolve_entity_set_async(items: List[uipath.platform.entities.entities.DataFabricEntityItem]) -> uipath.platform.entities.entities.EntitySetResolution # Retrieve an entity by its key. sdk.entities.retrieve(entity_key: str) -> uipath.platform.entities.entities.Entity @@ -511,11 +596,44 @@ sdk.entities.retrieve(entity_key: str) -> uipath.platform.entities.entities.Enti # Asynchronously retrieve an entity by its key. sdk.entities.retrieve_async(entity_key: str) -> uipath.platform.entities.entities.Entity +# Retrieve an entity by its name. +sdk.entities.retrieve_by_name(entity_name: str, folder_key: Optional[str]=None) -> uipath.platform.entities.entities.Entity + +# Asynchronously retrieve an entity by its name. +sdk.entities.retrieve_by_name_async(entity_name: str, folder_key: Optional[str]=None) -> uipath.platform.entities.entities.Entity + +# Retrieve records with structured filters, sorting, expansion, joins, and aggregates. +sdk.entities.retrieve_records(entity_key: str, filter_group: Optional[uipath.platform.entities.entities.EntityQueryFilterGroup]=None, sort_options: Optional[List[uipath.platform.entities.entities.EntityQuerySortOption]]=None, selected_fields: Optional[List[str]]=None, expansions: Optional[List[Any]]=None, expansion_level: Optional[int]=None, aggregates: Optional[List[uipath.platform.entities.entities.EntityAggregate]]=None, group_by: Optional[List[str]]=None, joins: Optional[List[uipath.platform.entities.entities.EntityJoin]]=None, binnings: Optional[List[uipath.platform.entities.entities.EntityBinning]]=None, start: Optional[int]=None, limit: Optional[int]=None) -> uipath.platform.entities.entities.RetrieveEntityRecordsResponse + +# Asynchronously retrieve records with structured filters, sorting, expansion, joins, and aggregates. +sdk.entities.retrieve_records_async(entity_key: str, filter_group: Optional[uipath.platform.entities.entities.EntityQueryFilterGroup]=None, sort_options: Optional[List[uipath.platform.entities.entities.EntityQuerySortOption]]=None, selected_fields: Optional[List[str]]=None, expansions: Optional[List[Any]]=None, expansion_level: Optional[int]=None, aggregates: Optional[List[uipath.platform.entities.entities.EntityAggregate]]=None, group_by: Optional[List[str]]=None, joins: Optional[List[uipath.platform.entities.entities.EntityJoin]]=None, binnings: Optional[List[uipath.platform.entities.entities.EntityBinning]]=None, start: Optional[int]=None, limit: Optional[int]=None) -> uipath.platform.entities.entities.RetrieveEntityRecordsResponse + +# Update an entity's display name, description, and/or RBAC flag. +sdk.entities.update_entity_metadata(entity_id: str, metadata: Union[uipath.platform.entities.entities.EntityMetadataUpdateOptions, Dict[str, Any]]) -> None + +# Asynchronously update an entity's display name, description, and/or RBAC flag. +sdk.entities.update_entity_metadata_async(entity_id: str, metadata: Union[uipath.platform.entities.entities.EntityMetadataUpdateOptions, Dict[str, Any]]) -> None + +# Update a single record by id and return the updated row. +sdk.entities.update_record(entity_key: str, record_id: str, data: Any, expansion_level: Optional[int]=None) -> uipath.platform.entities.entities.EntityRecord + +# Asynchronously update a single record by id. +sdk.entities.update_record_async(entity_key: str, record_id: str, data: Any, expansion_level: Optional[int]=None) -> uipath.platform.entities.entities.EntityRecord + # Update multiple records in an entity in a single batch operation. -sdk.entities.update_records(entity_key: str, records: List[Any], schema: Optional[Type[Any]]=None) -> uipath.platform.entities.entities.EntityRecordsBatchResponse +sdk.entities.update_records(entity_key: str, records: List[Any], schema: Optional[Type[Any]]=None, expansion_level: Optional[int]=None, fail_on_first: Optional[bool]=None) -> uipath.platform.entities.entities.EntityRecordsBatchResponse # Asynchronously update multiple records in an entity in a single batch operation. -sdk.entities.update_records_async(entity_key: str, records: List[Any], schema: Optional[Type[Any]]=None) -> uipath.platform.entities.entities.EntityRecordsBatchResponse +sdk.entities.update_records_async(entity_key: str, records: List[Any], schema: Optional[Type[Any]]=None, expansion_level: Optional[int]=None, fail_on_first: Optional[bool]=None) -> uipath.platform.entities.entities.EntityRecordsBatchResponse + +# Upload a file attachment to a File-type field on a record. +sdk.entities.upload_attachment(entity_id: str, record_id: str, field_name: str, file: Union[bytes, bytearray, memoryview, NoneType]=None, file_path: Optional[str]=None, expansion_level: Optional[int]=None) -> typing.Dict[str, typing.Any] + +# Asynchronously upload a file attachment to a File-type field on a record. +sdk.entities.upload_attachment_async(entity_id: str, record_id: str, field_name: str, file: Union[bytes, bytearray, memoryview, NoneType]=None, file_path: Optional[str]=None, expansion_level: Optional[int]=None) -> typing.Dict[str, typing.Any] + +# Parse a batch response, optionally validating success records against ``schema``. +sdk.entities.validate_entity_batch(batch_response: httpx.Response, schema: Optional[Type[Any]]=None) -> uipath.platform.entities.entities.EntityRecordsBatchResponse ``` @@ -619,6 +737,12 @@ sdk.jobs.retrieve_api_payload_async(inbox_id: str) -> typing.Any # Asynchronously retrieve a job identified by its key. sdk.jobs.retrieve_async(job_key: str, folder_key: str | None=None, folder_path: str | None=None, process_name: str | None=None) -> uipath.platform.orchestrator.job.Job +# Fetch payload data for Integration Services (Inbox) triggers. +sdk.jobs.retrieve_inbox_payload(inbox_id: str) -> typing.Any + +# Asynchronously fetch payload data for Integration Services (Inbox) triggers. +sdk.jobs.retrieve_inbox_payload_async(inbox_id: str) -> typing.Any + # Stop one or more jobs with specified strategy. sdk.jobs.stop(job_keys: List[str], strategy: str="SoftStop", folder_path: Optional[str]=None, folder_key: Optional[str]=None) -> None @@ -633,7 +757,7 @@ Llm service ```python # Generate chat completions using UiPath's normalized LLM Gateway API. -sdk.llm.chat_completions(messages: list[dict[str, str]] | list[tuple[str, str]], model: str="gpt-4.1-mini-2025-04-14", max_tokens: int=4096, temperature: float=0, n: int=1, frequency_penalty: float=0, presence_penalty: float=0, top_p: float | None=1, top_k: int | None=None, tools: list[uipath.platform.chat.llm_gateway.ToolDefinition] | None=None, tool_choice: Union[uipath.platform.chat.llm_gateway.AutoToolChoice, uipath.platform.chat.llm_gateway.RequiredToolChoice, uipath.platform.chat.llm_gateway.SpecificToolChoice, Literal['auto', 'none'], NoneType]=None, response_format: dict[str, Any] | type[pydantic.main.BaseModel] | None=None, api_version: str="2024-08-01-preview") +sdk.llm.chat_completions(messages: list[dict[str, str]] | list[tuple[str, str]], model: str="gpt-4.1-mini-2025-04-14", max_tokens: int=4096, temperature: float=0, n: int=1, frequency_penalty: float=0, presence_penalty: float=0, top_p: float | None=1, top_k: int | None=None, tools: list[uipath.platform.chat.llm_gateway.ToolDefinition | dict[str, Any]] | None=None, tool_choice: Union[uipath.platform.chat.llm_gateway.AutoToolChoice, uipath.platform.chat.llm_gateway.RequiredToolChoice, uipath.platform.chat.llm_gateway.SpecificToolChoice, Literal['auto', 'none'], NoneType]=None, response_format: dict[str, Any] | type[pydantic.main.BaseModel] | None=None, api_version: str="2024-08-01-preview") ``` @@ -669,6 +793,43 @@ sdk.mcp.retrieve_async(slug: str, folder_path: str | None=None) -> uipath.platfo ``` +### Memory + +Memory service + +```python +# Create a new memory space. +sdk.memory.create(name: str, description: Optional[str]=None, is_encrypted: Optional[bool]=None, folder_key: Optional[str]=None, folder_path: Optional[str]=None) -> uipath.platform.memory.memory.MemorySpace + +# Asynchronously create a new memory space. +sdk.memory.create_async(name: str, description: Optional[str]=None, is_encrypted: Optional[bool]=None, folder_key: Optional[str]=None, folder_path: Optional[str]=None) -> uipath.platform.memory.memory.MemorySpace + +# Ingest a resolved escalation outcome into memory. +sdk.memory.escalation_ingest(memory_space_id: str, request: uipath.platform.memory.memory.EscalationMemoryIngestRequest, folder_key: Optional[str]=None, folder_path: Optional[str]=None) -> None + +# Asynchronously ingest a resolved escalation outcome into memory. +sdk.memory.escalation_ingest_async(memory_space_id: str, request: uipath.platform.memory.memory.EscalationMemoryIngestRequest, folder_key: Optional[str]=None, folder_path: Optional[str]=None) -> None + +# Search escalation memory for previously resolved outcomes. +sdk.memory.escalation_search(memory_space_id: str, request: uipath.platform.memory.memory.MemorySearchRequest, folder_key: Optional[str]=None, folder_path: Optional[str]=None) -> uipath.platform.memory.memory.EscalationMemorySearchResponse + +# Asynchronously search escalation memory for previously resolved outcomes. +sdk.memory.escalation_search_async(memory_space_id: str, request: uipath.platform.memory.memory.MemorySearchRequest, folder_key: Optional[str]=None, folder_path: Optional[str]=None) -> uipath.platform.memory.memory.EscalationMemorySearchResponse + +# List memory spaces with optional OData query parameters. +sdk.memory.list(filter: Optional[str]=None, orderby: Optional[str]=None, top: Optional[int]=None, skip: Optional[int]=None, folder_key: Optional[str]=None, folder_path: Optional[str]=None) -> uipath.platform.memory.memory.MemorySpaceListResponse + +# Asynchronously list memory spaces. +sdk.memory.list_async(filter: Optional[str]=None, orderby: Optional[str]=None, top: Optional[int]=None, skip: Optional[int]=None, folder_key: Optional[str]=None, folder_path: Optional[str]=None) -> uipath.platform.memory.memory.MemorySpaceListResponse + +# Search a memory space via LLMOps. +sdk.memory.search(memory_space_id: str, request: uipath.platform.memory.memory.MemorySearchRequest, folder_key: Optional[str]=None, folder_path: Optional[str]=None) -> uipath.platform.memory.memory.MemorySearchResponse + +# Asynchronously search a memory space via LLMOps. +sdk.memory.search_async(memory_space_id: str, request: uipath.platform.memory.memory.MemorySearchRequest, folder_key: Optional[str]=None, folder_path: Optional[str]=None) -> uipath.platform.memory.memory.MemorySearchResponse + +``` + ### Orchestrator Setup Orchestrator Setup service @@ -682,16 +843,29 @@ sdk.orchestrator_setup.enable_first_run_async() -> None ``` +### Pii Detection + +Pii Detection service + +```python +# Detect PII in the provided documents and/or files. +sdk.pii_detection.detect_pii(request: uipath.platform.pii_detection.pii_detection.PiiDetectionRequest) -> uipath.platform.pii_detection.pii_detection.PiiDetectionResponse + +# Detect PII in the provided documents and/or files (async). +sdk.pii_detection.detect_pii_async(request: uipath.platform.pii_detection.pii_detection.PiiDetectionRequest) -> uipath.platform.pii_detection.pii_detection.PiiDetectionResponse + +``` + ### Processes Processes service ```python # Start execution of a process by its name. -sdk.processes.invoke(name: str, input_arguments: Optional[Dict[str, Any]]=None, folder_key: Optional[str]=None, folder_path: Optional[str]=None, attachments: Optional[list[uipath.platform.attachments.attachments.Attachment]]=None, parent_operation_id: Optional[str]=None, **kwargs) -> uipath.platform.orchestrator.job.Job +sdk.processes.invoke(name: str, input_arguments: Optional[Dict[str, Any]]=None, folder_key: Optional[str]=None, folder_path: Optional[str]=None, attachments: Optional[list[uipath.platform.attachments.attachments.Attachment]]=None, parent_operation_id: Optional[str]=None, run_as_me: Optional[bool]=None, **kwargs) -> uipath.platform.orchestrator.job.Job # Asynchronously start execution of a process by its name. -sdk.processes.invoke_async(name: str, input_arguments: Optional[Dict[str, Any]]=None, folder_key: Optional[str]=None, folder_path: Optional[str]=None, attachments: Optional[list[uipath.platform.attachments.attachments.Attachment]]=None, parent_operation_id: Optional[str]=None, **kwargs) -> uipath.platform.orchestrator.job.Job +sdk.processes.invoke_async(name: str, input_arguments: Optional[Dict[str, Any]]=None, folder_key: Optional[str]=None, folder_path: Optional[str]=None, attachments: Optional[list[uipath.platform.attachments.attachments.Attachment]]=None, parent_operation_id: Optional[str]=None, run_as_me: Optional[bool]=None, **kwargs) -> uipath.platform.orchestrator.job.Job ``` @@ -766,19 +940,32 @@ Resource Catalog service sdk.resource_catalog.list(resource_types: Optional[List[uipath.platform.resource_catalog.resource_catalog.ResourceType]]=None, resource_sub_types: Optional[List[str]]=None, folder_path: Optional[str]=None, folder_key: Optional[str]=None, page_size: int=20) -> typing.Iterator[uipath.platform.resource_catalog.resource_catalog.Resource] # Asynchronously get tenant scoped resources and folder scoped resources (accessible to the user). -sdk.resource_catalog.list_async(resource_types: Optional[List[uipath.platform.resource_catalog.resource_catalog.ResourceType]]=None, resource_sub_types: Optional[List[str]]=None, folder_path: Optional[str]=None, folder_key: Optional[str]=None, page_size: int=20) -> typing.AsyncIterator[uipath.platform.resource_catalog.resource_catalog.Resource] +sdk.resource_catalog.list_async(resource_types: Optional[List[uipath.platform.resource_catalog.resource_catalog.ResourceType]]=None, resource_sub_types: Optional[List[str]]=None, folder_path: Optional[str]=None, folder_key: Optional[str]=None, page_size: int=20) -> typing.AsyncGenerator[uipath.platform.resource_catalog.resource_catalog.Resource, NoneType] # Get resources of a specific type (tenant scoped or folder scoped). sdk.resource_catalog.list_by_type(resource_type: typing.Iterator[uipath.platform.resource_catalog.resource_catalog.Resource] # Asynchronously get resources of a specific type (tenant scoped or folder scoped). -sdk.resource_catalog.list_by_type_async(resource_type: typing.AsyncIterator[uipath.platform.resource_catalog.resource_catalog.Resource] +sdk.resource_catalog.list_by_type_async(resource_type: typing.AsyncGenerator[uipath.platform.resource_catalog.resource_catalog.Resource, NoneType] # Search for tenant scoped resources and folder scoped resources (accessible to the user). sdk.resource_catalog.search(name: Optional[str]=None, resource_types: Optional[List[uipath.platform.resource_catalog.resource_catalog.ResourceType]]=None, resource_sub_types: Optional[List[str]]=None, page_size: int=20) -> typing.Iterator[uipath.platform.resource_catalog.resource_catalog.Resource] # Asynchronously search for tenant scoped resources and folder scoped resources (accessible to the user). -sdk.resource_catalog.search_async(name: Optional[str]=None, resource_types: Optional[List[uipath.platform.resource_catalog.resource_catalog.ResourceType]]=None, resource_sub_types: Optional[List[str]]=None, page_size: int=20) -> typing.AsyncIterator[uipath.platform.resource_catalog.resource_catalog.Resource] +sdk.resource_catalog.search_async(name: Optional[str]=None, resource_types: Optional[List[uipath.platform.resource_catalog.resource_catalog.ResourceType]]=None, resource_sub_types: Optional[List[str]]=None, page_size: int=20) -> typing.AsyncGenerator[uipath.platform.resource_catalog.resource_catalog.Resource, NoneType] + +``` + +### Semantic Proxy + +Semantic Proxy service + +```python +# Detect PII in the provided documents and/or files. +sdk.semantic_proxy.detect_pii(request: uipath.platform.semantic_proxy.semantic_proxy.PiiDetectionRequest) -> uipath.platform.semantic_proxy.semantic_proxy.PiiDetectionResponse + +# Detect PII in the provided documents and/or files (async). +sdk.semantic_proxy.detect_pii_async(request: uipath.platform.semantic_proxy.semantic_proxy.PiiDetectionRequest) -> uipath.platform.semantic_proxy.semantic_proxy.PiiDetectionResponse ``` @@ -793,6 +980,12 @@ sdk.tasks.create(title: str, data: Optional[Dict[str, Any]]=None, app_name: Opti # Creates a new action asynchronously. sdk.tasks.create_async(title: str, data: Optional[Dict[str, Any]]=None, app_name: Optional[str]=None, app_key: Optional[str]=None, app_folder_path: Optional[str]=None, app_folder_key: Optional[str]=None, assignee: Optional[str]=None, recipient: Optional[uipath.platform.action_center.tasks.TaskRecipient]=None, priority: Optional[str]=None, labels: Optional[List[str]]=None, is_actionable_message_enabled: Optional[bool]=None, actionable_message_metadata: Optional[Dict[str, Any]]=None, source_name: str="Agent") -> uipath.platform.action_center.tasks.Task +# Create a new QuickForm task synchronously. +sdk.tasks.create_quickform(title: str, task_schema_key: str, schema: Dict[str, Any], data: Optional[Dict[str, Any]]=None, folder_path: Optional[str]=None, folder_key: Optional[str]=None, assignee: Optional[str]=None, recipient: Optional[uipath.platform.action_center.tasks.TaskRecipient]=None, priority: Optional[str]=None, labels: Optional[List[str]]=None, is_actionable_message_enabled: Optional[bool]=None, actionable_message_metadata: Optional[Dict[str, Any]]=None, creator_job_key: Optional[str]=None, source_name: str="Agent") -> uipath.platform.action_center.tasks.Task + +# Creates a new QuickForm task asynchronously. +sdk.tasks.create_quickform_async(title: str, task_schema_key: str, schema: Dict[str, Any], data: Optional[Dict[str, Any]]=None, folder_path: Optional[str]=None, folder_key: Optional[str]=None, assignee: Optional[str]=None, recipient: Optional[uipath.platform.action_center.tasks.TaskRecipient]=None, priority: Optional[str]=None, labels: Optional[List[str]]=None, is_actionable_message_enabled: Optional[bool]=None, actionable_message_metadata: Optional[Dict[str, Any]]=None, creator_job_key: Optional[str]=None, source_name: str="Agent") -> uipath.platform.action_center.tasks.Task + # Retrieves a task by its key synchronously. sdk.tasks.retrieve(action_key: str, app_folder_path: Optional[str]=None, app_folder_key: Optional[str]=None, app_name: str | None=None) -> uipath.platform.action_center.tasks.Task diff --git a/packages/uipath/src/uipath/_utils/_auth.py b/packages/uipath/src/uipath/_utils/_auth.py index 6f83fd0a2..b13ab5316 100644 --- a/packages/uipath/src/uipath/_utils/_auth.py +++ b/packages/uipath/src/uipath/_utils/_auth.py @@ -4,7 +4,8 @@ from pathlib import Path from typing import Optional -from .constants import ( +from uipath.platform.constants import ( + DOTENV_FILE, ENV_BASE_URL, ENV_UIPATH_ACCESS_TOKEN, ENV_UNATTENDED_USER_ACCESS_TOKEN, @@ -22,7 +23,7 @@ def parse_access_token(access_token: str): def update_env_file(env_contents): - env_path = Path.cwd() / ".env" + env_path = Path.cwd() / DOTENV_FILE if env_path.exists(): with open(env_path, "r") as f: for line in f: diff --git a/packages/uipath/src/uipath/_utils/_request_override.py b/packages/uipath/src/uipath/_utils/_request_override.py index 07d5e2ebc..872ce79ed 100644 --- a/packages/uipath/src/uipath/_utils/_request_override.py +++ b/packages/uipath/src/uipath/_utils/_request_override.py @@ -1,7 +1,11 @@ from base64 import b64encode from typing import Optional -from .constants import HEADER_FOLDER_KEY, HEADER_FOLDER_PATH, HEADER_FOLDER_PATH_ENCODED +from uipath.platform.constants import ( + HEADER_FOLDER_KEY, + HEADER_FOLDER_PATH, + HEADER_FOLDER_PATH_ENCODED, +) def folder_path_header(folder_path: str) -> dict[str, str]: diff --git a/packages/uipath/src/uipath/_utils/_user_agent.py b/packages/uipath/src/uipath/_utils/_user_agent.py index dcf28d3e0..1083faa45 100644 --- a/packages/uipath/src/uipath/_utils/_user_agent.py +++ b/packages/uipath/src/uipath/_utils/_user_agent.py @@ -1,6 +1,6 @@ import importlib -from .constants import HEADER_USER_AGENT +from uipath.platform.constants import HEADER_USER_AGENT def user_agent_value(specific_component: str) -> str: diff --git a/packages/uipath/src/uipath/_utils/constants.py b/packages/uipath/src/uipath/_utils/constants.py index e5a2c06da..a50459a4b 100644 --- a/packages/uipath/src/uipath/_utils/constants.py +++ b/packages/uipath/src/uipath/_utils/constants.py @@ -1,89 +1,16 @@ -# Environment variables -DOTENV_FILE = ".env" -ENV_BASE_URL = "UIPATH_URL" -ENV_EVAL_BACKEND_URL = "UIPATH_EVAL_BACKEND_URL" -ENV_UNATTENDED_USER_ACCESS_TOKEN = "UNATTENDED_USER_ACCESS_TOKEN" -ENV_UIPATH_ACCESS_TOKEN = "UIPATH_ACCESS_TOKEN" -ENV_FOLDER_KEY = "UIPATH_FOLDER_KEY" -ENV_FOLDER_PATH = "UIPATH_FOLDER_PATH" -ENV_JOB_KEY = "UIPATH_JOB_KEY" -ENV_JOB_ID = "UIPATH_JOB_ID" -ENV_ROBOT_KEY = "UIPATH_ROBOT_KEY" -ENV_TENANT_ID = "UIPATH_TENANT_ID" -ENV_TENANT_NAME = "UIPATH_TENANT_NAME" -ENV_ORGANIZATION_ID = "UIPATH_ORGANIZATION_ID" -ENV_TELEMETRY_ENABLED = "UIPATH_TELEMETRY_ENABLED" -ENV_TRACING_ENABLED = "UIPATH_TRACING_ENABLED" -ENV_UIPATH_PROJECT_ID = "UIPATH_PROJECT_ID" -ENV_PROJECT_KEY = "PROJECT_KEY" -ENV_PROCESS_KEY = "UIPATH_PROCESS_KEY" -ENV_UIPATH_PROCESS_UUID = "UIPATH_PROCESS_UUID" -ENV_UIPATH_TRACE_ID = "UIPATH_TRACE_ID" -ENV_UIPATH_PROCESS_VERSION = "UIPATH_PROCESS_VERSION" +"""Deprecated alias for the canonical constants module. -# Headers -HEADER_AGENTHUB_CONFIG = "x-uipath-agenthub-config" -HEADER_FOLDER_KEY = "x-uipath-folderkey" -HEADER_FOLDER_PATH = "x-uipath-folderpath" -HEADER_FOLDER_PATH_ENCODED = "x-uipath-folderpath-encoded" -HEADER_INTERNAL_ACCOUNT_ID = "x-uipath-internal-accountid" -HEADER_INTERNAL_TENANT_ID = "x-uipath-internal-tenantid" -HEADER_JOB_KEY = "x-uipath-jobkey" -HEADER_LLMGATEWAY_BYO_CONNECTION_ID = "x-uipath-llmgateway-byoisconnectionid" -HEADER_PROCESS_KEY = "x-uipath-processkey" -HEADER_SW_LOCK_KEY = "x-uipath-sw-lockkey" -HEADER_TENANT_ID = "x-uipath-tenantid" -HEADER_TRACE_ID = "x-uipath-traceid" -HEADER_USER_AGENT = "x-uipath-user-agent" +This module is kept as a backward-compatibility shim so existing imports keep +working. New code should import from ``uipath.platform.constants``. +""" -# Data sources (request types) -ORCHESTRATOR_STORAGE_BUCKET_DATA_SOURCE_REQUEST = ( - "#UiPath.Vdbs.Domain.Api.V20Models.StorageBucketDataSourceRequest" -) -CONFLUENCE_DATA_SOURCE_REQUEST = ( - "#UiPath.Vdbs.Domain.Api.V20Models.ConfluenceDataSourceRequest" -) -DROPBOX_DATA_SOURCE_REQUEST = ( - "#UiPath.Vdbs.Domain.Api.V20Models.DropboxDataSourceRequest" -) -GOOGLE_DRIVE_DATA_SOURCE_REQUEST = ( - "#UiPath.Vdbs.Domain.Api.V20Models.GoogleDriveDataSourceRequest" -) -ONEDRIVE_DATA_SOURCE_REQUEST = ( - "#UiPath.Vdbs.Domain.Api.V20Models.OneDriveDataSourceRequest" -) - -# Data sources -ORCHESTRATOR_STORAGE_BUCKET_DATA_SOURCE = ( - "#UiPath.Vdbs.Domain.Api.V20Models.StorageBucketDataSource" -) -CONFLUENCE_DATA_SOURCE = "#UiPath.Vdbs.Domain.Api.V20Models.ConfluenceDataSource" -DROPBOX_DATA_SOURCE = "#UiPath.Vdbs.Domain.Api.V20Models.DropboxDataSource" -GOOGLE_DRIVE_DATA_SOURCE = "#UiPath.Vdbs.Domain.Api.V20Models.GoogleDriveDataSource" -ONEDRIVE_DATA_SOURCE = "#UiPath.Vdbs.Domain.Api.V20Models.OneDriveDataSource" +import warnings as _warnings -# Preprocessing request types -LLMV3Mini_REQUEST = "#UiPath.Vdbs.Domain.Api.V20Models.LLMV3MiniPreProcessingRequest" -LLMV4_REQUEST = "#UiPath.Vdbs.Domain.Api.V20Models.LLMV4PreProcessingRequest" -NativeV1_REQUEST = "#UiPath.Vdbs.Domain.Api.V20Models.NativeV1PreProcessingRequest" +from uipath.platform.constants import * # noqa: F401,F403 - -# Local storage -TEMP_ATTACHMENTS_FOLDER = "uipath_attachments" - -# LLM models -COMMUNITY_agents_SUFFIX = "-community-agents" - -# File names -PYTHON_CONFIGURATION_FILE = "pyproject.toml" -UIPATH_CONFIG_FILE = "uipath.json" -UIPATH_BINDINGS_FILE = "bindings.json" -ENTRY_POINTS_FILE = "entry-points.json" -STUDIO_METADATA_FILE = "studio_metadata.json" - - -# Folder names -LEGACY_EVAL_FOLDER = "evals" -EVALS_FOLDER = "evaluations" -# Evaluators -CUSTOM_EVALUATOR_PREFIX = "file://" +_warnings.warn( + "uipath._utils.constants is deprecated and will be removed in a future release; " + "import from uipath.platform.constants instead.", + FutureWarning, + stacklevel=2, +) diff --git a/packages/uipath/src/uipath/agent/models/agent.py b/packages/uipath/src/uipath/agent/models/agent.py index 901ce7ca9..86a0de739 100644 --- a/packages/uipath/src/uipath/agent/models/agent.py +++ b/packages/uipath/src/uipath/agent/models/agent.py @@ -35,6 +35,7 @@ ) from uipath.eval.mocks import ExampleCall from uipath.platform.connections import Connection +from uipath.platform.entities import DataFabricEntityItem, DataFabricOntologyItem from uipath.platform.guardrails import ( BuiltInValidatorGuardrail, ) @@ -111,9 +112,12 @@ class AgentToolType(str, CaseInsensitiveEnum): PROCESS = "Process" API = "Api" PROCESS_ORCHESTRATION = "ProcessOrchestration" + FLOW = "Flow" + FUNCTION = "Function" INTEGRATION = "Integration" INTERNAL = "Internal" IXP = "Ixp" + CLIENT_SIDE = "ClientSide" UNKNOWN = "Unknown" # fallback branch discriminator @@ -123,6 +127,7 @@ class AgentInternalToolType(str, CaseInsensitiveEnum): ANALYZE_FILES = "analyze-attachments" DEEP_RAG = "deep-rag" BATCH_TRANSFORM = "batch-transform" + HTTP_REQUEST = "http-request" class AgentEscalationRecipientType(str, CaseInsensitiveEnum): @@ -134,6 +139,18 @@ class AgentEscalationRecipientType(str, CaseInsensitiveEnum): ASSET_USER_EMAIL = "AssetUserEmail" GROUP_NAME = "GroupName" ASSET_GROUP_NAME = "AssetGroupName" + ARGUMENT_EMAIL = "ArgumentEmail" + ARGUMENT_GROUP_NAME = "ArgumentGroupName" + WORKLOAD = "Workload" + ROUND_ROBIN = "RoundRobin" + CUSTOM_ASSIGNEES = "CustomAssignees" + + +class AgentEscalationChannelType(str, CaseInsensitiveEnum): + """Agent escalation channel type enumeration.""" + + ACTION_CENTER = "actionCenter" + ACTION_CENTER_QUICK_FORM = "actionCenterQuickForm" class AgentContextRetrievalMode(str, CaseInsensitiveEnum): @@ -153,6 +170,7 @@ class AgentContextType(str, CaseInsensitiveEnum): INDEX = "index" ATTACHMENTS = "attachments" DATA_FABRIC_ENTITY_SET = "datafabricentityset" + DATA_FABRIC_ONTOLOGY = "datafabricontology" class AgentMessageRole(str, CaseInsensitiveEnum): @@ -162,6 +180,12 @@ class AgentMessageRole(str, CaseInsensitiveEnum): USER = "user" +class AgentVariant(str, CaseInsensitiveEnum): + """Agent variant enumeration.""" + + CASE_MANAGER = "caseManager" + + class AgentGuardrailActionType(str, CaseInsensitiveEnum): """Agent guardrail action type enumeration.""" @@ -179,6 +203,7 @@ class AgentToolArgumentPropertiesVariant(str, CaseInsensitiveEnum): ARGUMENT = "argument" STATIC = "static" TEXT_BUILDER = "textBuilder" + ARRAY_BUILDER = "arrayBuilder" class TextTokenType(str, CaseInsensitiveEnum): @@ -267,11 +292,21 @@ class AgentToolTextBuilderArgumentProperties(BaseAgentToolArgumentProperties): tokens: List[TextToken] +class AgentToolArrayBuilderArgumentProperties(BaseCfg): + """Agent array builder argument properties model.""" + + variant: Literal[AgentToolArgumentPropertiesVariant.ARRAY_BUILDER] = Field( + default=AgentToolArgumentPropertiesVariant.ARRAY_BUILDER, + frozen=True, + ) + + AgentToolArgumentProperties = Annotated[ Union[ AgentToolStaticArgumentProperties, AgentToolArgumentArgumentProperties, AgentToolTextBuilderArgumentProperties, + AgentToolArrayBuilderArgumentProperties, ], Field(discriminator="variant"), _case_insensitive_enum_validator("variant", AgentToolArgumentPropertiesVariant), @@ -394,16 +429,6 @@ class AgentContextSettings(BaseCfg): ) -class DataFabricEntityItem(BaseCfg): - """A single Data Fabric entity reference.""" - - id: str - reference_key: Optional[str] = Field(None, alias="referenceKey") - name: str - folder_id: str = Field(alias="folderId") - description: Optional[str] = None - - class AgentContextResourceConfig(BaseAgentResourceConfig): """Agent context resource configuration model.""" @@ -417,6 +442,9 @@ class AgentContextResourceConfig(BaseAgentResourceConfig): None, description="Context settings" ) entity_set: Optional[List[DataFabricEntityItem]] = Field(None, alias="entitySet") + ontology_set: Optional[List[DataFabricOntologyItem]] = Field( + None, alias="ontologySet" + ) argument_properties: Dict[str, AgentToolArgumentProperties] = Field( {}, alias="argumentProperties" ) @@ -426,6 +454,11 @@ def is_datafabric(self) -> bool: """Check if this context is a Data Fabric entity set resource.""" return self.context_type == AgentContextType.DATA_FABRIC_ENTITY_SET + @property + def is_datafabric_ontology(self) -> bool: + """Check if this context is a Data Fabric ontology resource.""" + return self.context_type == AgentContextType.DATA_FABRIC_ONTOLOGY + @property def datafabric_entity_identifiers(self) -> list[str]: """Extract entity identifiers from entitySet.""" @@ -447,13 +480,57 @@ class AgentMcpTool(BaseCfg): class DynamicToolsMode(str, CaseInsensitiveEnum): - """Dynamic tools mode enumeration.""" + """Dynamic tools mode enumeration. + + Deprecated: kept for backwards compatibility with older ``agent.json`` files + that still serialize the ``dynamicTools`` field. New code should use + :class:`ToolsConfiguration` (see ``AgentMcpResourceConfig.tools_configuration``). + """ NONE = "none" SCHEMA = "schema" ALL = "all" +class CachedToolsConfig(BaseCfg): + """Cached tools configuration: use the tools saved in the agent definition snapshot. + + When ``refresh_schema_before_call`` is true, the live tool schema is fetched + from the MCP server immediately before a tool is invoked. The agent still uses + the cached schema to decide which tool to call; the fresh schema is applied only + at invocation time. + """ + + type: Literal["cached"] = Field(default="cached", frozen=True) + refresh_schema_before_call: bool = Field( + default=True, alias="refreshSchemaBeforeCall" + ) + + +class DynamicToolsConfig(BaseCfg): + """Dynamic tools configuration: fetch the tool list from the MCP server at runtime. + + When ``allow_all`` is true, every tool the server exposes is forwarded + to the agent. When false, the live list is filtered by the snapshot's + ``available_tools`` allowlist (live schemas, curated tool set). + """ + + type: Literal["dynamic"] = Field(default="dynamic", frozen=True) + allow_all: bool = Field(alias="allowAll") + + +DiscoveryMode = Annotated[ + Union[CachedToolsConfig, DynamicToolsConfig], + Field(discriminator="type"), +] + + +class ToolsConfiguration(BaseCfg): + """Configuration describing how tools are sourced for an MCP resource.""" + + discovery_mode: DiscoveryMode = Field(alias="discoveryMode") + + class AgentMcpResourceConfig(BaseAgentResourceConfig): """Agent MCP resource configuration model.""" @@ -463,8 +540,8 @@ class AgentMcpResourceConfig(BaseAgentResourceConfig): folder_path: str = Field(alias="folderPath") slug: str = Field(..., alias="slug") available_tools: List[AgentMcpTool] = Field(..., alias="availableTools") - dynamic_tools: DynamicToolsMode = Field( - default=DynamicToolsMode.NONE, alias="dynamicTools" + tools_configuration: Optional[ToolsConfiguration] = Field( + default=None, alias="toolsConfiguration" ) @@ -476,15 +553,10 @@ class AgentA2aResourceConfig(BaseAgentResourceConfig): ) id: str slug: str = Field(..., alias="slug") - agent_card_url: str = Field(default="", alias="agentCardUrl") - is_active: bool = Field(default=True, alias="isActive") + folder_path: str = Field(alias="folderPath") cached_agent_card: Optional[Dict[str, Any]] = Field( default=None, alias="cachedAgentCard" ) - created_at: Optional[str] = Field(default=None, alias="createdAt") - created_by: Optional[str] = Field(default=None, alias="createdBy") - updated_at: Optional[str] = Field(default=None, alias="updatedAt") - updated_by: Optional[str] = Field(default=None, alias="updatedBy") _RECIPIENT_TYPE_NORMALIZED_MAP: Mapping[int | str, AgentEscalationRecipientType] = { @@ -495,6 +567,11 @@ class AgentA2aResourceConfig(BaseAgentResourceConfig): 5: AgentEscalationRecipientType.GROUP_NAME, "staticgroupname": AgentEscalationRecipientType.GROUP_NAME, 6: AgentEscalationRecipientType.ASSET_GROUP_NAME, + 7: AgentEscalationRecipientType.ARGUMENT_EMAIL, + 8: AgentEscalationRecipientType.ARGUMENT_GROUP_NAME, + 9: AgentEscalationRecipientType.WORKLOAD, + 10: AgentEscalationRecipientType.ROUND_ROBIN, + 11: AgentEscalationRecipientType.CUSTOM_ASSIGNEES, } @@ -550,9 +627,129 @@ class AssetRecipient(BaseEscalationRecipient): folder_path: str = Field(..., alias="folderPath") +class ArgumentEmailRecipient(BaseEscalationRecipient): + """Argument email recipient resolved from a named input argument. + + The argument_path supports dot-notation for nested input fields (e.g. "user.email"). + """ + + type: Literal[AgentEscalationRecipientType.ARGUMENT_EMAIL,] = Field( + ..., alias="type" + ) + argument_path: str = Field(..., alias="argumentName") + + +class ArgumentGroupNameRecipient(BaseEscalationRecipient): + """Argument group name recipient resolved from a named input argument. + + The argument_path supports dot-notation for nested input fields (e.g. "team.groupName"). + """ + + type: Literal[AgentEscalationRecipientType.ARGUMENT_GROUP_NAME,] = Field( + ..., alias="type" + ) + argument_path: str = Field(..., alias="argumentName") + + +class WorkloadRecipient(BaseEscalationRecipient): + """Workload-based group assignment. + + The Action Center distributes tasks to the group member with the lightest workload. + """ + + type: Literal[AgentEscalationRecipientType.WORKLOAD,] = Field(..., alias="type") + value: str = Field(..., alias="value") + display_name: str = Field(..., alias="displayName") + + +class RoundRobinRecipient(BaseEscalationRecipient): + """Round-robin group assignment. + + The Action Center cycles through group members in order on each new task. + """ + + type: Literal[AgentEscalationRecipientType.ROUND_ROBIN,] = Field(..., alias="type") + value: str = Field(..., alias="value") + display_name: str = Field(..., alias="displayName") + + +class CustomAssigneesRecipient(BaseEscalationRecipient): + """Custom multi-user assignment. + + A channel can carry multiple instances, one per assignee email. All are passed + to Action Center together using a Workload assignment criteria. + """ + + type: Literal[AgentEscalationRecipientType.CUSTOM_ASSIGNEES,] = Field( + ..., alias="type" + ) + value: str = Field(..., alias="value") + display_name: Optional[str] = Field(default=None, alias="displayName") + + +class ToolOutputRecipient(BaseEscalationRecipient): + """Recipient whose value is resolved at runtime from a named tool's output. + + Instead of a literal value entered at design time, this binding points at a + field within a named tool's output. The runtime walks the agent's message + history, finds the most recent ToolMessage matching `tool_name`, parses its + content as JSON, and extracts `output_path` (a top-level field for v1). + + Only the assignment-criteria recipient types that accept a runtime-computed + value are supported: USER_ID, GROUP_ID, WORKLOAD, ROUND_ROBIN, + CUSTOM_ASSIGNEES. The asset/static/argument types do not participate in + tool-output binding (they have their own design-time resolution rules). + """ + + type: Literal[ + AgentEscalationRecipientType.USER_ID, + AgentEscalationRecipientType.GROUP_ID, + AgentEscalationRecipientType.WORKLOAD, + AgentEscalationRecipientType.ROUND_ROBIN, + AgentEscalationRecipientType.CUSTOM_ASSIGNEES, + ] = Field(..., alias="type") + source: Literal["toolOutput"] = Field(..., alias="source") + tool_name: str = Field(..., alias="toolName") + output_path: str = Field(..., alias="outputPath") + + +# ────────────────────────────────────────────────────────────────────────────── +# AgentEscalationRecipient — Union ordering & invariants +# ────────────────────────────────────────────────────────────────────────────── +# Pydantic evaluates Union members left-to-right and stops at the first +# successful match, so member order determines which class a payload resolves +# to when multiple members share the same `type` value (e.g. WORKLOAD is valid +# on both WorkloadRecipient and ToolOutputRecipient). +# +# How dispatching works: +# - Payload with `source: "toolOutput"` → matches ToolOutputRecipient +# (it is the only class declaring `source` as a required Literal field). +# - Payload without `source` → ToolOutputRecipient validation fails +# (`source` missing), so it falls through to the literal class below +# that owns its `type`. +# +# Why we don't use `Field(discriminator="type")`: +# The `type` values are NOT unique across the Union — both WorkloadRecipient +# and ToolOutputRecipient declare `type=WORKLOAD`, same for the other +# tool-output-capable criteria. A typed discriminator requires unique +# discriminator values across members, which this union violates by design. +# +# Critical invariants (any of these breaking causes silent mis-typing): +# 1. ToolOutputRecipient remains the FIRST member of the Union. +# 2. ToolOutputRecipient.source remains a required `Literal["toolOutput"]` +# (NOT `Optional`, NOT a default value). +# 3. No literal class below it gains an optional `source` field. AgentEscalationRecipient = Annotated[ - Union[StandardRecipient, AssetRecipient], - Field(discriminator="type"), + Union[ + ToolOutputRecipient, + StandardRecipient, + AssetRecipient, + ArgumentEmailRecipient, + ArgumentGroupNameRecipient, + WorkloadRecipient, + RoundRobinRecipient, + CustomAssigneesRecipient, + ], BeforeValidator(_normalize_recipient_type), ] @@ -617,13 +814,9 @@ def _resolve_task_title(v: Any) -> Any: return v -class AgentEscalationChannelProperties(BaseResourceProperties): - """Agent escalation channel properties model.""" +class BaseEscalationChannelProperties(BaseResourceProperties): + """Fields shared by every escalation channel's properties.""" - app_name: str | None = Field(default=None, alias="appName") - app_version: int = Field(..., alias="appVersion") - folder_name: Optional[str] = Field(None, alias="folderName") - resource_key: str | None = Field(default=None, alias="resourceKey") is_actionable_message_enabled: Optional[bool] = Field( None, alias="isActionableMessageEnabled" ) @@ -632,12 +825,31 @@ class AgentEscalationChannelProperties(BaseResourceProperties): ) -class AgentEscalationChannel(BaseCfg): - """Agent escalation channel model.""" +class AgentEscalationChannelProperties(BaseEscalationChannelProperties): + """Action Center app-task channel properties (channel type ``actionCenter``).""" + + app_name: str | None = Field(default=None, alias="appName") + app_version: int = Field(..., alias="appVersion") + folder_name: Optional[str] = Field(None, alias="folderName") + resource_key: str | None = Field(default=None, alias="resourceKey") + + +class AgentQuickFormChannelProperties(BaseEscalationChannelProperties): + """Quick Form channel properties (channel type ``actionCenterQuickForm``).""" + + form_schema: Dict[str, Any] = Field(..., alias="schema") + + @property + def schema_id(self) -> str | None: + """Return the schema id nested inside the form schema body.""" + return self.form_schema.get("schemaId") + + +class BaseAgentEscalationChannel(BaseCfg): + """Fields shared by every escalation channel variant.""" id: Optional[str] = Field(None, alias="id") name: str = Field(..., alias="name") - type: str = Field(alias="type") description: str = Field(..., alias="description") input_schema: Dict[str, Any] = Field(..., alias="inputSchema") output_schema: Dict[str, Any] = Field(EMPTY_SCHEMA, alias="outputSchema") @@ -645,7 +857,6 @@ class AgentEscalationChannel(BaseCfg): {}, alias="argumentProperties" ) outcome_mapping: Optional[Dict[str, str]] = Field(None, alias="outcomeMapping") - properties: AgentEscalationChannelProperties = Field(..., alias="properties") recipients: List[AgentEscalationRecipient] = Field(..., alias="recipients") task_title: Optional[Union[str, TaskTitle]] = Field( default="Escalation Task", alias="taskTitle" @@ -660,6 +871,34 @@ def _apply_task_title_resolution(cls, v: Any) -> Any: return _resolve_task_title(v) +class AgentEscalationChannel(BaseAgentEscalationChannel): + """Action Center app-task escalation channel (channel type ``actionCenter``).""" + + type: Literal[AgentEscalationChannelType.ACTION_CENTER] = Field( + default=AgentEscalationChannelType.ACTION_CENTER, alias="type" + ) + properties: AgentEscalationChannelProperties = Field(..., alias="properties") + + +class AgentQuickFormEscalationChannel(BaseAgentEscalationChannel): + """Quick Form escalation channel; FormLib schema lives in ``properties.form_schema``.""" + + type: Literal[AgentEscalationChannelType.ACTION_CENTER_QUICK_FORM] = Field( + default=AgentEscalationChannelType.ACTION_CENTER_QUICK_FORM, alias="type" + ) + properties: AgentQuickFormChannelProperties = Field(..., alias="properties") + + +EscalationChannel = Annotated[ + Union[ + AgentEscalationChannel, + AgentQuickFormEscalationChannel, + ], + Field(discriminator="type"), + _case_insensitive_enum_validator("type", AgentEscalationChannelType), +] + + class AgentEscalationResourceConfig(BaseAgentResourceConfig): """Agent escalation resource configuration model.""" @@ -667,7 +906,7 @@ class AgentEscalationResourceConfig(BaseAgentResourceConfig): resource_type: Literal[AgentResourceType.ESCALATION] = Field( alias="$resourceType", default=AgentResourceType.ESCALATION, frozen=True ) - channels: List[AgentEscalationChannel] = Field(alias="channels") + channels: List[EscalationChannel] = Field(alias="channels") is_agent_memory_enabled: bool = Field(default=False, alias="isAgentMemoryEnabled") escalation_type: Literal[0] = Field(default=0, alias="escalationType") @@ -687,7 +926,7 @@ class AgentIxpVsEscalationResourceConfig(BaseAgentResourceConfig): resource_type: Literal[AgentResourceType.ESCALATION] = Field( alias="$resourceType", default=AgentResourceType.ESCALATION, frozen=True ) - channels: List[AgentEscalationChannel] = Field(alias="channels") + channels: List[EscalationChannel] = Field(alias="channels") is_agent_memory_enabled: bool = Field(default=False, alias="isAgentMemoryEnabled") escalation_type: Literal[1] = Field(default=1, alias="escalationType") vs_escalation_properties: AgentIxpVsEscalationProperties = Field( @@ -719,6 +958,8 @@ class AgentProcessToolResourceConfig(BaseAgentToolResourceConfig): AgentToolType.PROCESS, AgentToolType.API, AgentToolType.PROCESS_ORCHESTRATION, + AgentToolType.FLOW, + AgentToolType.FUNCTION, ] output_schema: Dict[str, Any] = Field(EMPTY_SCHEMA, alias="outputSchema") properties: AgentProcessToolProperties @@ -806,11 +1047,20 @@ class AgentInternalBatchTransformToolProperties(BaseResourceProperties): settings: AgentInternalBatchTransformSettings = Field(..., alias="settings") +class AgentInternalHttpRequestToolProperties(BaseResourceProperties): + """Agent internal http request tool properties model.""" + + tool_type: Literal[AgentInternalToolType.HTTP_REQUEST] = Field( + alias="toolType", default=AgentInternalToolType.HTTP_REQUEST, frozen=True + ) + + AgentInternalToolProperties = Annotated[ Union[ AgentInternalAnalyzeFilesToolProperties, AgentInternalDeepRagToolProperties, AgentInternalBatchTransformToolProperties, + AgentInternalHttpRequestToolProperties, ], Field(discriminator="tool_type"), _case_insensitive_enum_validator("tool_type", AgentInternalToolType, "toolType"), @@ -870,6 +1120,15 @@ class AgentInternalToolResourceConfig(BaseAgentToolResourceConfig): ) +class AgentClientSideToolResourceConfig(BaseAgentToolResourceConfig): + """Resource config for client-side tools executed by the client SDK.""" + + type: Literal[AgentToolType.CLIENT_SIDE] = AgentToolType.CLIENT_SIDE + properties: BaseResourceProperties = Field(default_factory=BaseResourceProperties) + output_schema: Optional[Dict[str, Any]] = Field(None, alias="outputSchema") + arguments: Optional[Dict[str, Any]] = Field(default_factory=dict) + + class AgentUnknownToolResourceConfig(BaseAgentToolResourceConfig): """Fallback for unknown tool types (parent normalizer sets type='Unknown').""" @@ -883,11 +1142,13 @@ class AgentUnknownToolResourceConfig(BaseAgentToolResourceConfig): AgentIntegrationToolResourceConfig, AgentInternalToolResourceConfig, AgentIxpExtractionResourceConfig, + AgentClientSideToolResourceConfig, AgentUnknownToolResourceConfig, # when parent sets type="Unknown" ], Field(discriminator="type"), ] + EscalationResourceConfig = Annotated[ Union[ Annotated[AgentEscalationResourceConfig, Tag(0)], @@ -1137,6 +1398,7 @@ class AgentMetadata(BaseCfg): """Agent metadata model.""" is_conversational: bool = Field(alias="isConversational") + variant: Optional[AgentVariant] = Field(default=None, alias="variant") storage_version: str = Field(alias="storageVersion") @@ -1197,6 +1459,13 @@ def is_conversational(self) -> bool: return metadata.is_conversational return False + @property + def is_case_manager(self) -> bool: + """Checks if the agent is a case manager agent.""" + if not self.metadata: + return False + return self.metadata.variant == AgentVariant.CASE_MANAGER + @staticmethod def _normalize_guardrails(v: Dict[str, Any]) -> None: guards = v.get("guardrails") @@ -1254,9 +1523,12 @@ def _normalize_resources(v: Dict[str, Any]) -> None: "process": "Process", "api": "Api", "processorchestration": "ProcessOrchestration", + "flow": "Flow", + "function": "Function", "integration": "Integration", "internal": "Internal", "ixp": "Ixp", + "clientside": "ClientSide", "unknown": "Unknown", } CONTEXT_MODE_MAP = { diff --git a/packages/uipath/src/uipath/agent/react/__init__.py b/packages/uipath/src/uipath/agent/react/__init__.py index 835fd0bda..539f062b6 100644 --- a/packages/uipath/src/uipath/agent/react/__init__.py +++ b/packages/uipath/src/uipath/agent/react/__init__.py @@ -6,15 +6,18 @@ from .conversational_prompts import ( PromptUserSettings, get_chat_system_prompt, + get_generate_output_prompt, ) from .conversational_voice_prompts import get_voice_system_prompt from .prompts import AGENT_SYSTEM_PROMPT_TEMPLATE from .tools import ( END_EXECUTION_TOOL, RAISE_ERROR_TOOL, + SET_CONVERSATIONAL_OUTPUT_TOOL, EndExecutionToolSchemaModel, FlowControlToolConfig, RaiseErrorToolSchemaModel, + SetConversationalOutputToolSchemaModel, ) __all__ = [ @@ -22,9 +25,12 @@ "FlowControlToolConfig", "END_EXECUTION_TOOL", "RAISE_ERROR_TOOL", + "SET_CONVERSATIONAL_OUTPUT_TOOL", "EndExecutionToolSchemaModel", "RaiseErrorToolSchemaModel", + "SetConversationalOutputToolSchemaModel", "PromptUserSettings", "get_chat_system_prompt", + "get_generate_output_prompt", "get_voice_system_prompt", ] diff --git a/packages/uipath/src/uipath/agent/react/conversational_prompts.py b/packages/uipath/src/uipath/agent/react/conversational_prompts.py index 7732c7f69..d966dd55b 100644 --- a/packages/uipath/src/uipath/agent/react/conversational_prompts.py +++ b/packages/uipath/src/uipath/agent/react/conversational_prompts.py @@ -62,6 +62,8 @@ class PromptUserSettings(BaseModel): - Never attempt calls with incomplete data - On errors: modify parameters or change approach (never retry identical calls) +{{CONVERSATIONAL_AGENT_SERVICE_PREFIX_conversationIdPrompt}} + ===================================================================== TOOL RESULTS ===================================================================== @@ -136,18 +138,26 @@ class PromptUserSettings(BaseModel): {user_settings_json} ```""" +_CONVERSATION_ID_TEMPLATE = """ +The current conversation ID is {conversation_id}. This may be useful to include in tool-calls when tool parameters specify passing in the conversation ID. Other than tool-call inputs, this ID should not be mentioned to the user. +""" + def get_chat_system_prompt( model: str, system_message: str, agent_name: str | None, user_settings: PromptUserSettings | None = None, + conversation_id: str | None = None, ) -> str: """Generate a system prompt for a conversational agent. Args: - agent_definition: Conversational agent definition + model: Model identifier. + system_message: The agent system prompt content. + agent_name: The agent display name; defaults to "Unnamed Agent" when None. user_settings: Optional user data that is injected into the system prompt. + conversation_id: Optional conversation identifier that is injected into the system prompt. Returns: The complete system prompt string @@ -177,6 +187,10 @@ def get_chat_system_prompt( "{{CONVERSATIONAL_AGENT_SERVICE_PREFIX_userSettingsPrompt}}", get_user_settings_template(user_settings), ) + prompt = prompt.replace( + "{{CONVERSATIONAL_AGENT_SERVICE_PREFIX_conversationIdPrompt}}", + get_conversation_id_template(conversation_id), + ) return prompt @@ -190,7 +204,7 @@ def get_user_settings_template( user_settings: User profile information Returns: - The user context template with JSON or empty string + The filled-in user settings template if user_settings is provided, otherwise an empty string """ if user_settings is None: return "" @@ -205,3 +219,35 @@ def get_user_settings_template( user_settings_json = json.dumps(settings_dict, ensure_ascii=False) return _USER_CONTEXT_TEMPLATE.format(user_settings_json=user_settings_json) + + +def get_conversation_id_template(conversation_id: str | None) -> str: + """Get the conversation ID prompt section. + + Args: + conversation_id: The ID of the current conversation, if any + + Returns: + The filled-in conversation ID template if conversation_id is provided, otherwise an empty string + """ + if not conversation_id: + return "" + return _CONVERSATION_ID_TEMPLATE.format(conversation_id=conversation_id) + + +_GENERATE_OUTPUT_INSTRUCTION = """The conversational response for this turn has already been delivered to the user. Call the `set_conversational_output` tool to record the structured output fields for this turn. + +Rules: +- For each field, use values inferred from the conversation's recent turn. +- For optional fields that are not yet relevant or determinable (e.g. the conversation is still gathering context, or the topic hasn't surfaced yet), omit them entirely. +- For required fields that cannot yet be determined, provide a default placeholder. DO NOT fabricate, guess, or hallucinate meaningful values. +- Do not produce any text response, as this will not be seen by the user. Only call the tool.""" + + +def get_generate_output_prompt() -> str: + """Return the framework-internal generate-output instruction. + + Appended as a final user-message to the conversational structured-output + node's LLM call. + """ + return _GENERATE_OUTPUT_INSTRUCTION diff --git a/packages/uipath/src/uipath/agent/react/tools.py b/packages/uipath/src/uipath/agent/react/tools.py index c2a162be9..050653543 100644 --- a/packages/uipath/src/uipath/agent/react/tools.py +++ b/packages/uipath/src/uipath/agent/react/tools.py @@ -11,6 +11,7 @@ class FlowControlToolName(str, Enum): END_EXECUTION = "end_execution" RAISE_ERROR = "raise_error" + SET_CONVERSATIONAL_OUTPUT = "set_conversational_output" @dataclass(frozen=True) @@ -72,3 +73,25 @@ class RaiseErrorToolSchemaModel(BaseModel): description="Raises an error and ends the execution of the agent", args_schema=RaiseErrorToolSchemaModel, ) + + +class SetConversationalOutputToolSchemaModel(BaseModel): + """Placeholder args_schema for the `set_conversational_output` tool. + + Always overridden at construction time with the agent's stripped output + schema (i.e. the user's `outputSchema` with `uipath__agent_response_messages` + removed). Declared here so the tool entry has a well-typed default. + """ + + model_config = ConfigDict(extra="forbid") + + +SET_CONVERSATIONAL_OUTPUT_TOOL = FlowControlToolConfig( + name=FlowControlToolName.SET_CONVERSATIONAL_OUTPUT, + description=( + "Sets the structured output fields for the current conversational " + "turn. Called once per turn after the conversational response has been " + "delivered, to populate fields as the agent's output." + ), + args_schema=SetConversationalOutputToolSchemaModel, +) diff --git a/packages/uipath/src/uipath/eval/_helpers/evaluators_helpers.py b/packages/uipath/src/uipath/eval/_helpers/evaluators_helpers.py index d64954b99..577c61ce7 100644 --- a/packages/uipath/src/uipath/eval/_helpers/evaluators_helpers.py +++ b/packages/uipath/src/uipath/eval/_helpers/evaluators_helpers.py @@ -1,5 +1,6 @@ import ast import json +import re from collections.abc import Mapping, Sequence from datetime import datetime from typing import Any @@ -11,6 +12,19 @@ ToolOutput, ) +TOOL_NAME_ATTR = "tool.name" + +# Mirrors uipath_langchain.agent.tools.utils.sanitize_tool_name; pinned by TestSanitizedNameMatch. +_TOOL_NAME_DISALLOWED = re.compile(r"[^a-zA-Z0-9_-]") + + +def _sanitize_tool_name(name: str | None) -> str: + """Sanitise a tool name the same way the LangChain runtime does.""" + if not name: + return "" + return _TOOL_NAME_DISALLOWED.sub("", "_".join(name.split()))[:64] + + COMPARATOR_MAPPINGS = { ">": "gt", "<": "lt", @@ -21,14 +35,90 @@ "!=": "ne", } -COMMUNITY_agents_SUFFIX = "-community-agents" + +def _unsynthesized_tool_attrs(span: ReadableSpan) -> Mapping[str, Any] | None: + """Return span.attributes if this is a real tool invocation, else None.""" + attrs = span.attributes + if ( + not attrs + or attrs.get("tool.synthesized", False) + or not attrs.get(TOOL_NAME_ATTR) + ): + return None + return attrs + + +def _match_key(actual_name: str, actual_id: str | None, expected_key: str) -> bool: + """Strict per-call kind: id-only when actual has one, sanitised-name otherwise — never cross-kind.""" + if actual_id is not None: + return expected_key == actual_id + return _sanitize_tool_name(expected_key) == _sanitize_tool_name(actual_name) + + +def _calls_match(actual, expected) -> bool: + """Strict per-call kind: id-only when actual has one, sanitised-name otherwise — never cross-kind.""" + if actual.id is not None: + # Picker stores the id under `expected.name` when an id was chosen — honour either field. + expected_key = expected.id if expected.id is not None else expected.name + return actual.id == expected_key + return _sanitize_tool_name(actual.name) == _sanitize_tool_name(expected.name) + + +def _parse_tool_args(input_value: Any) -> dict[str, Any]: + """Coerce a span's `input.value` into a dict of tool args. + + Tries JSON first (handles `true`/`false`/`null` and double-quoted keys), + falls back to `ast.literal_eval` for Python literal syntax (single-quoted + dict repr). Returns `{}` for non-dict parsed values or any parse failure. + """ + if isinstance(input_value, dict): + return input_value + if not isinstance(input_value, str): + return {} + try: + try: + parsed = json.loads(input_value) + except ValueError: # JSONDecodeError is a ValueError + parsed = ast.literal_eval(input_value) + except (SyntaxError, ValueError): + return {} + return parsed if isinstance(parsed, dict) else {} + + +def _tool_id_from(attrs: Mapping[str, Any]) -> str | None: + """Return the span's `tool.id` as a string when present, else None. + + Uses `is not None` (not truthiness) so an id of 0 or '' isn't dropped. + """ + tool_id = attrs.get("tool.id") + return str(tool_id) if tool_id is not None else None + + +def _build_tool_call(span: ReadableSpan, include_args: bool) -> ToolCall | None: + """Build a ToolCall from a span, or None for synthesized / non-tool spans.""" + attrs = _unsynthesized_tool_attrs(span) + if attrs is None: + return None + tool_name = str(attrs[TOOL_NAME_ATTR]) + tool_id = _tool_id_from(attrs) + args = _parse_tool_args(attrs.get("input.value", {})) if include_args else {} + return ToolCall(name=tool_name, args=args, id=tool_id) + + +def count_tool_calls_by_name_and_id(tool_calls: Sequence[ToolCall]) -> dict[str, int]: + """Bucket each call under its id when present, else its name — strict per-call kind, no cross-kind matching.""" + counts: dict[str, int] = {} + for c in tool_calls: + key = c.id if c.id is not None else c.name + counts[key] = counts.get(key, 0) + 1 + return counts def extract_tool_calls_names(spans: Sequence[ReadableSpan]) -> list[str]: """Extract the tool call names from execution spans IN ORDER. Args: - spans: List of ReadableSpan objects from agent execution. + spans: List of ReadableSpan objects from workload execution. Returns: List of tool names in the order they were called. @@ -36,48 +126,36 @@ def extract_tool_calls_names(spans: Sequence[ReadableSpan]) -> list[str]: tool_calls_names = [] for span in spans: - # Check for tool.name attribute first - if span.attributes and (tool_name := span.attributes.get("tool.name")): - tool_calls_names.append(str(tool_name)) + if (attrs := _unsynthesized_tool_attrs(span)) is not None: + tool_calls_names.append(str(attrs[TOOL_NAME_ATTR])) return tool_calls_names -def extract_tool_calls(spans: Sequence[ReadableSpan]) -> list[ToolCall]: - """Extract the tool calls from execution spans with their arguments. +def extract_tool_calls( + spans: Sequence[ReadableSpan], + include_args: bool = True, +) -> list[ToolCall]: + """Extract the tool calls from execution spans. Args: - spans: List of ReadableSpan objects from agent execution. + spans: List of ReadableSpan objects from workload execution. + include_args: When False, skip parsing `input.value` and return + ToolCall objects with `args={}`. Use for evaluators that only + need name/id (count, order) — avoids a parse per span on large + traces. Returns: - Dict of tool calls with their arguments. + List of tool calls with their arguments. """ - tool_calls = [] - - for span in spans: - if span.attributes and (tool_name := span.attributes.get("tool.name")): - try: - input_value: Any = span.attributes.get("input.value", {}) - # Ensure input_value is a string before parsing - if isinstance(input_value, str): - arguments = ast.literal_eval(input_value) - elif isinstance(input_value, dict): - arguments = input_value - else: - arguments = {} - tool_calls.append(ToolCall(name=str(tool_name), args=arguments)) - except (json.JSONDecodeError, SyntaxError, ValueError): - # Handle case where input.value is not valid JSON/Python syntax - tool_calls.append(ToolCall(name=str(tool_name), args={})) - - return tool_calls + return [c for s in spans if (c := _build_tool_call(s, include_args)) is not None] def extract_tool_calls_outputs(spans: Sequence[ReadableSpan]) -> list[ToolOutput]: """Extract the outputs of the tool calls from execution spans. Args: - spans: List of ReadableSpan objects from agent execution. + spans: List of ReadableSpan objects from workload execution. Returns: List of tool calls outputs. @@ -87,8 +165,10 @@ def extract_tool_calls_outputs(spans: Sequence[ReadableSpan]) -> list[ToolOutput potential_output_keys = ["content"] tool_calls_outputs = [] for span in spans: - if span.attributes and (tool_name := span.attributes.get("tool.name")): - output = span.attributes.get("output.value", "") + if (attrs := _unsynthesized_tool_attrs(span)) is not None: + tool_name = str(attrs[TOOL_NAME_ATTR]) + tool_id = _tool_id_from(attrs) + output = attrs.get("output.value", "") final_output = "" # Handle different output formats @@ -118,8 +198,9 @@ def extract_tool_calls_outputs(spans: Sequence[ReadableSpan]) -> list[ToolOutput tool_calls_outputs.append( ToolOutput( - name=str(tool_name), + name=tool_name, output=str(final_output) if final_output else "", + id=tool_id, ) ) return tool_calls_outputs @@ -196,6 +277,105 @@ def tool_calls_order_score( return lcs_length / n, justification +def _strict_order_score( + actual: Sequence[ToolCall], + expected: Sequence[str], + justification: dict[str, Any], +) -> tuple[float, dict[str, Any]]: + """Strict-mode evaluation — only an exact positional match scores 1.0.""" + if len(actual) != len(expected): + return 0.0, justification + for i, key in enumerate(expected): + if not _match_key(actual[i].name, actual[i].id, key): + return 0.0, justification + justification["lcs"] = list(expected) + return 1.0, justification + + +def _build_lcs_dp( + actual: Sequence[ToolCall], expected: Sequence[str] +) -> list[list[int]]: + """Fill the LCS dynamic-programming table for id-aware matching.""" + m, n = len(actual), len(expected) + dp = [[0] * (n + 1) for _ in range(m + 1)] + for i in range(1, m + 1): + for j in range(1, n + 1): + if _match_key(actual[i - 1].name, actual[i - 1].id, expected[j - 1]): + dp[i][j] = dp[i - 1][j - 1] + 1 + else: + dp[i][j] = max(dp[i - 1][j], dp[i][j - 1]) + return dp + + +def _reconstruct_lcs( + actual: Sequence[ToolCall], + expected: Sequence[str], + dp: list[list[int]], +) -> list[str]: + """Walk the DP table backwards to recover the LCS as a list of expected keys.""" + lcs: list[str] = [] + i, j = len(actual), len(expected) + while i > 0 and j > 0: + if _match_key(actual[i - 1].name, actual[i - 1].id, expected[j - 1]): + lcs.append(expected[j - 1]) + i -= 1 + j -= 1 + elif dp[i - 1][j] > dp[i][j - 1]: + i -= 1 + else: + j -= 1 + lcs.reverse() + return lcs + + +def tool_calls_order_score_with_ids( + actual_tool_calls: Sequence[ToolCall], + expected_tool_calls_keys: Sequence[str], + strict: bool = False, +) -> tuple[float, dict[str, Any]]: + """LCS-based ordering score with id-aware matching. + + Identical scoring algorithm to `tool_calls_order_score`, but each expected + key string is allowed to match either the actual call's `id` or its + `name`. Use this when eval-set criteria may be authored against the + stable tool id so renames of `name` don't silently break ordering checks. + + Args: + actual_tool_calls: ToolCall objects in the actual order. Each may carry + an `id` from the runtime's `tool.id` span attribute. + expected_tool_calls_keys: List of names OR ids in the expected order. + strict: When True, only perfect matches score above 0. + + Returns: + Same shape as `tool_calls_order_score`. The "actual" justification + renders the resolved match-key sequence (id when available, else name) + so the LCS reconstruction reads clearly. + """ + actual_keys: list[str] = [ + (c.id if c.id is not None else c.name) for c in actual_tool_calls + ] + justification: dict[str, Any] = { + "actual": str(list(actual_keys)), + "expected": str(list(expected_tool_calls_keys)), + "lcs": [], + } + + if not expected_tool_calls_keys and not actual_tool_calls: + return 1.0, justification + if not expected_tool_calls_keys or not actual_tool_calls: + return 0.0, justification + + if strict: + return _strict_order_score( + actual_tool_calls, expected_tool_calls_keys, justification + ) + + dp = _build_lcs_dp(actual_tool_calls, expected_tool_calls_keys) + lcs = _reconstruct_lcs(actual_tool_calls, expected_tool_calls_keys, dp) + justification["lcs"] = lcs + return len(lcs) / len(expected_tool_calls_keys), justification + + def tool_calls_count_score( actual_tool_calls_count: Mapping[str, int], expected_tool_calls_count: Mapping[str, tuple[str, int]], @@ -240,7 +420,12 @@ def tool_calls_count_score( expected_comparator, expected_count, ) in expected_tool_calls_count.items(): - actual_count = actual_tool_calls_count.get(tool_name, 0.0) + # Raw key first (id-keyed / exact-match), then sanitised (legacy display-name). `is None` not `or`: count of 0 is a hit. + actual_count = actual_tool_calls_count.get(tool_name) + if actual_count is None: + actual_count = actual_tool_calls_count.get( + _sanitize_tool_name(tool_name), 0 + ) comparator = f"__{COMPARATOR_MAPPINGS[expected_comparator]}__" to_add = float(getattr(actual_count, comparator)(expected_count)) @@ -310,7 +495,7 @@ def tool_calls_args_score( for expected_tool_call in expected_tool_calls: for idx, call in enumerate(actual_tool_calls): - if call.name == expected_tool_call.name and idx not in visited: + if _calls_match(call, expected_tool_call) and idx not in visited: # Get or initialize counter for this tool name tool_counters[call.name] = tool_counters.get(call.name, 0) tool_key = f"{call.name}_{tool_counters[call.name]}" @@ -402,7 +587,7 @@ def tool_calls_output_score( for idx, actual_tool_call_output in enumerate(actual_tool_calls_outputs): if idx in visited: continue - if actual_tool_call_output.name == expected_tool_call_output.name: + if _calls_match(actual_tool_call_output, expected_tool_call_output): # Get or initialize counter for this tool name tool_counters[actual_tool_call_output.name] = tool_counters.get( actual_tool_call_output.name, 0 @@ -449,23 +634,23 @@ def tool_calls_output_score( ), justifications -def trace_to_str(agent_trace: Sequence[ReadableSpan]) -> str: - """Convert OTEL spans to a platform-style agent run history string. +def trace_to_str(workload_trace: Sequence[ReadableSpan]) -> str: + """Convert OTEL spans to a platform-style workload run history string. Creates a similar structure to LangChain message processing but using OTEL spans. Only processes tool spans (spans with 'tool.name' attribute). Args: - agent_trace: List of ReadableSpan objects from the agent execution + workload_trace: List of ReadableSpan objects from the workload execution Returns: - String representation of the agent run history in platform format + String representation of the workload run history in platform format """ platform_history = [] seen_tool_calls = set() - for span in agent_trace: - if span.attributes and (tool_name := span.attributes.get("tool.name")): + for span in workload_trace: + if span.attributes and (tool_name := span.attributes.get(TOOL_NAME_ATTR)): # Get span timing information start_time = span.start_time end_time = span.end_time diff --git a/packages/uipath/src/uipath/eval/_helpers/output_path.py b/packages/uipath/src/uipath/eval/_helpers/output_path.py index 5c50a34df..0278b5153 100644 --- a/packages/uipath/src/uipath/eval/_helpers/output_path.py +++ b/packages/uipath/src/uipath/eval/_helpers/output_path.py @@ -1,4 +1,4 @@ -"""Utility for resolving dot-notation paths from agent output dictionaries. +"""Utility for resolving dot-notation paths from workload output dictionaries. Supports: - "*" → return entire output (default behavior) diff --git a/packages/uipath/src/uipath/eval/evaluators/_aggregator_specs.py b/packages/uipath/src/uipath/eval/evaluators/_aggregator_specs.py new file mode 100644 index 000000000..d3d44cc36 --- /dev/null +++ b/packages/uipath/src/uipath/eval/evaluators/_aggregator_specs.py @@ -0,0 +1,62 @@ +"""Aggregator specs embedded in per-datapoint classification evaluator configs. + +Each aggregator is a run-level metric (precision / recall / f-score) attached +to a classification evaluator. Classes are declared once on the parent evaluator +config — every aggregator on the same evaluator operates on the same class +vocabulary, so the field is not repeated per spec. Only the metric-shape fields +(``averaging`` and, for fscore, ``f_value``) live on the spec itself. +""" + +from __future__ import annotations + +from typing import Annotated, Literal, Union + +from pydantic import BaseModel, ConfigDict, Field +from pydantic.alias_generators import to_camel + + +class _AggregatorSpecBase(BaseModel): + """Shared pydantic config for every aggregator variant.""" + + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + + +class PrecisionAggregatorSpec(_AggregatorSpecBase): + """Run-level precision aggregator (multiclass, micro or macro averaged).""" + + type: Literal["precision"] = "precision" + averaging: Literal["macro", "micro"] + + +class RecallAggregatorSpec(_AggregatorSpecBase): + """Run-level recall aggregator (multiclass, micro or macro averaged).""" + + type: Literal["recall"] = "recall" + averaging: Literal["macro", "micro"] + + +class FScoreAggregatorSpec(_AggregatorSpecBase): + """Run-level F-beta aggregator (multiclass, micro or macro averaged).""" + + type: Literal["fscore"] = "fscore" + averaging: Literal["macro", "micro"] + # Upper bound keeps beta² finite — a huge beta overflows to inf and the + # F-score becomes NaN, which is not representable in JSON. + f_value: float = Field(default=1.0, gt=0, le=1000) + + +class ConfusionMatrixAggregatorSpec(_AggregatorSpecBase): + """Run-level raw k×k confusion matrix — no scalar headline, no averaging.""" + + type: Literal["confusion_matrix"] = "confusion_matrix" + + +AggregatorSpec = Annotated[ + Union[ + PrecisionAggregatorSpec, + RecallAggregatorSpec, + FScoreAggregatorSpec, + ConfusionMatrixAggregatorSpec, + ], + Field(discriminator="type"), +] diff --git a/packages/uipath/src/uipath/eval/evaluators/base_dataset_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/base_dataset_evaluator.py new file mode 100644 index 000000000..5f302b076 --- /dev/null +++ b/packages/uipath/src/uipath/eval/evaluators/base_dataset_evaluator.py @@ -0,0 +1,45 @@ +"""Base abstractions for dataset-level evaluators. + +A dataset-level evaluator runs once per evaluation set, after all per-datapoint +evaluators have produced their results. It consumes the per-datapoint +EvaluationResultDto values from one named source evaluator and emits a single +EvaluationResult that summarizes the dataset. + +Concretely distinct from GenericBaseEvaluator: different evaluate() signature, +different lifecycle. Kept as a parallel hierarchy rather than a subclass so the +runtime cannot accidentally dispatch a dataset evaluator through the +per-datapoint loop. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod + +from ..models.models import EvaluationResult, EvaluationResultDto +from ._aggregator_specs import AggregatorSpec + + +class BaseDatasetEvaluator(ABC): + """Abstract base for dataset-level evaluators. + + Constructed from an :class:`AggregatorSpec`, the source evaluator's name, + and the class vocabulary of the parent per-datapoint evaluator. Classes + live on the evaluator config (not the spec) — every aggregator on the same + evaluator operates on the same vocabulary. + """ + + spec: AggregatorSpec + source_evaluator: str + classes: list[str] + + def __init__( + self, spec: AggregatorSpec, source_evaluator: str, classes: list[str] + ) -> None: + """Store the aggregator spec, source evaluator name, and shared classes.""" + self.spec = spec + self.source_evaluator = source_evaluator + self.classes = classes + + @abstractmethod + def evaluate(self, results: list[EvaluationResultDto]) -> EvaluationResult: + """Reduce per-datapoint results into a single run-level EvaluationResult.""" diff --git a/packages/uipath/src/uipath/eval/evaluators/base_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/base_evaluator.py index 73fac46c6..9bb4a4a6e 100644 --- a/packages/uipath/src/uipath/eval/evaluators/base_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/base_evaluator.py @@ -8,7 +8,7 @@ from pydantic.alias_generators import to_camel from .._helpers.helpers import track_evaluation_metrics -from ..models import AgentExecution, EvaluationResult +from ..models import EvaluationResult, WorkloadExecution from ..models.models import ( EvaluationResultDto, UiPathEvaluationError, @@ -47,6 +47,22 @@ class BaseEvaluatorJustification(BaseModel): expected: str actual: str + @classmethod + def try_from(cls, details: object) -> "BaseEvaluatorJustification | None": + """Coerce a free-form details payload into a justification, or return None. + + Accepts either an existing instance or a dict that ``model_validate`` can + parse. Anything else (str, None, malformed dict) yields ``None``. + """ + if isinstance(details, cls): + return details + if isinstance(details, dict): + try: + return cls.model_validate(details) + except Exception: + return None + return None + # Additional type variables for Config and Justification # Note: C must be BaseEvaluatorConfig[T] to ensure type consistency @@ -575,22 +591,22 @@ def reduce_scores(self, results: list[EvaluationResultDto]) -> float: @abstractmethod async def validate_and_evaluate_criteria( - self, agent_execution: AgentExecution, evaluation_criteria: Any + self, workload_execution: WorkloadExecution, evaluation_criteria: Any ) -> EvaluationResult: """Evaluate the given data and return a result from a raw evaluation criteria.""" pass @abstractmethod async def evaluate( - self, agent_execution: AgentExecution, evaluation_criteria: T + self, workload_execution: WorkloadExecution, evaluation_criteria: T ) -> EvaluationResult: """Evaluate the given data and return a result. Args: - agent_execution: The execution details containing: + workload_execution: The execution details containing: - agent_input: The input received by the agent - - agent_output: The actual output from the agent - - agent_trace: The execution trace from the agent + - workload_output: The actual output from the agent + - workload_trace: The execution trace from the agent - simulation_instructions: The simulation instructions for the agent evaluation_criteria: The criteria to evaluate @@ -626,7 +642,7 @@ def model_post_init(self, __context: Any) -> None: self.description = self.evaluator_config.description async def validate_and_evaluate_criteria( - self, agent_execution: AgentExecution, evaluation_criteria: Any + self, workload_execution: WorkloadExecution, evaluation_criteria: Any ) -> EvaluationResult: """Evaluate the given data and return a result from a raw evaluation criteria.""" if evaluation_criteria is None: @@ -639,4 +655,4 @@ async def validate_and_evaluate_criteria( category=UiPathEvaluationErrorCategory.SYSTEM, ) criteria = self.validate_evaluation_criteria(evaluation_criteria) - return await self.evaluate(agent_execution, criteria) + return await self.evaluate(workload_execution, criteria) diff --git a/packages/uipath/src/uipath/eval/evaluators/base_legacy_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/base_legacy_evaluator.py index d53dcda20..10b4b8f55 100644 --- a/packages/uipath/src/uipath/eval/evaluators/base_legacy_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/base_legacy_evaluator.py @@ -10,10 +10,10 @@ from ..models import EvaluationResult from ..models.models import ( - AgentExecution, ErrorEvaluationResult, LegacyEvaluatorCategory, LegacyEvaluatorType, + WorkloadExecution, ) from .attachment_utils import ( download_attachment_as_string, @@ -111,7 +111,7 @@ def get_evaluator_id(cls) -> str: async def validate_and_evaluate_criteria( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: LegacyEvaluationCriteria, ) -> EvaluationResult: """Evaluate the given data and return a result from a raw evaluation criteria.""" @@ -119,19 +119,19 @@ async def validate_and_evaluate_criteria( # Check if line-by-line evaluation is enabled if self.line_by_line_evaluation: - return await self._evaluate_line_by_line(agent_execution, criteria) + return await self._evaluate_line_by_line(workload_execution, criteria) - return await self.evaluate(agent_execution, criteria) + return await self.evaluate(workload_execution, criteria) async def _evaluate_line_by_line( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: LegacyEvaluationCriteria, ) -> EvaluationResult: """Evaluate output line-by-line and aggregate results. Args: - agent_execution: The execution details + workload_execution: The execution details evaluation_criteria: The evaluation criteria Returns: @@ -140,7 +140,7 @@ async def _evaluate_line_by_line( from .line_by_line_utils import build_line_by_line_result, evaluate_lines # Extract actual and expected outputs - actual_output = self._get_actual_output(agent_execution) + actual_output = self._get_actual_output(workload_execution) expected_output = evaluation_criteria.expected_output # Split into lines using utility function @@ -168,7 +168,7 @@ def create_line_criteria(expected_line: str) -> LegacyEvaluationCriteria: actual_lines=actual_lines, expected_lines=expected_lines, target_output_key=self.target_output_key, - agent_execution=agent_execution, + workload_execution=workload_execution, evaluate_fn=self.evaluate, create_line_criteria_fn=create_line_criteria, ) @@ -181,29 +181,32 @@ def create_line_criteria(expected_line: str) -> LegacyEvaluationCriteria: expected_lines=expected_lines, ) - def _get_actual_output(self, agent_execution: AgentExecution) -> Any: - """Extract actual output from agent execution. + def _get_actual_output(self, workload_execution: WorkloadExecution) -> Any: + """Extract actual output from workload execution. If the output is a job attachment URI, downloads the attachment and returns its content as a string. Args: - agent_execution: The agent execution + workload_execution: The workload execution Returns: - The actual output (either the full agent_output or a specific key) + The actual output (either the full workload_output or a specific key) """ - agent_output = agent_execution.agent_output + workload_output = workload_execution.workload_output # If target_output_key is "*", return full output if self.target_output_key == "*": - result = agent_output + result = workload_output # Otherwise, extract specific key - elif isinstance(agent_output, dict) and self.target_output_key in agent_output: - result = agent_output[self.target_output_key] + elif ( + isinstance(workload_output, dict) + and self.target_output_key in workload_output + ): + result = workload_output[self.target_output_key] else: # Fallback to full output - result = agent_output + result = workload_output # Check if result is a job attachment URI and download if so if is_job_attachment_uri(result): @@ -217,13 +220,13 @@ def _get_actual_output(self, agent_execution: AgentExecution) -> Any: @abstractmethod async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: LegacyEvaluationCriteria, ) -> EvaluationResult: """Evaluate the given data and return a result. Args: - agent_execution: The execution details containing: + workload_execution: The execution details containing: - agent_input: The input received by the agent - actual_output: The actual output from the agent - spans: The execution spans to use for the evaluation diff --git a/packages/uipath/src/uipath/eval/evaluators/binary_classification_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/binary_classification_evaluator.py index d56509228..051de81a3 100644 --- a/packages/uipath/src/uipath/eval/evaluators/binary_classification_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/binary_classification_evaluator.py @@ -1,4 +1,4 @@ -"""Binary classification evaluator for agent outputs. +"""Binary classification evaluator for workload outputs. Evaluates binary classification by comparing predicted vs expected class. Per-datapoint score is 1.0 (correct) or 0.0 (incorrect). The reduce_scores @@ -9,10 +9,10 @@ from typing import Literal from ..models import ( - AgentExecution, EvaluationResult, EvaluatorType, NumericEvaluationResult, + WorkloadExecution, ) from ..models.models import ( EvaluationResultDto, @@ -63,11 +63,11 @@ def get_evaluator_id(cls) -> str: async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: BinaryClassificationEvaluationCriteria, ) -> EvaluationResult: """Evaluate binary classification by comparing predicted vs expected class.""" - predicted_class = str(self._get_actual_output(agent_execution)).lower() + predicted_class = str(self._get_actual_output(workload_execution)).lower() expected_class = evaluation_criteria.expected_class.lower() positive_class = self.evaluator_config.positive_class.lower() diff --git a/packages/uipath/src/uipath/eval/evaluators/classification_dataset_evaluators.py b/packages/uipath/src/uipath/eval/evaluators/classification_dataset_evaluators.py new file mode 100644 index 000000000..dd8367610 --- /dev/null +++ b/packages/uipath/src/uipath/eval/evaluators/classification_dataset_evaluators.py @@ -0,0 +1,270 @@ +"""Dataset-level classification evaluators: Precision, Recall, F-score, Confusion Matrix. + +All variants share the same internal machinery — a k x k confusion matrix built +from each per-datapoint result's BaseEvaluatorJustification (expected, actual) +strings. The scalar variants (precision / recall / fscore) emit per-class +metrics plus micro/macro averages and pick the headline ``score`` per the +spec's ``averaging``; the ``confusion_matrix`` variant emits only the raw grid +with a 0.0 placeholder score. + +The ``details`` payload is the platform wire contract: the Agents reducer +worker (python-dataset-eval-worker) calls this evaluator and ships +``details.model_dump(by_alias=True, exclude_none=True)`` verbatim to the C# +backend, where the frontend's zod schema +(frontend-sw/src/schemas/evaluations/evals.ts) validates it. Changing field +names or shapes here is a cross-repo breaking change. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from pydantic import BaseModel, ConfigDict, Field +from pydantic.alias_generators import to_camel + +from ..models.models import ( + EvaluationResult, + EvaluationResultDto, + NumericEvaluationResult, +) +from ._aggregator_specs import ( + ConfusionMatrixAggregatorSpec, + FScoreAggregatorSpec, +) +from .base_dataset_evaluator import BaseDatasetEvaluator +from .base_evaluator import BaseEvaluatorJustification + + +class PerClassMetrics(BaseModel): + """Per-class confusion counts plus all three scalar metrics for that class.""" + + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + + tp: int + tn: int + fp: int + fn: int + support: int + precision: float + recall: float + f_score: float + + +class AveragedMetrics(BaseModel): + """Micro- or macro-averaged precision / recall / F-score triple.""" + + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + + precision: float + recall: float + f_score: float + + +class ClassificationDetails(BaseModel): + """Structured details payload emitted by every classification aggregator. + + The scalar metrics (precision / recall / fscore) populate every field; + the ``confusion_matrix`` variant emits only the grid + counts, leaving + ``averaging`` / ``f_value`` / ``per_class`` / ``macro`` / ``micro`` as + None (excluded from the wire dump). + """ + + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + + metric: str + classes: list[str] + confusion_matrix: list[list[int]] = Field( + ..., + description=( + "k x k confusion matrix indexed as " + "``confusion_matrix[predicted_idx][expected_idx]`` " + "(rows are predicted classes, columns are expected). " + "This is the transpose of sklearn's convention " + "(``[true][predicted]``); UI / consumer code must use the " + "orientation documented here." + ), + ) + n_total: int + n_scored: int + n_skipped: int + averaging: str | None = None + f_value: float | None = None + per_class: dict[str, PerClassMetrics] | None = None + macro: AveragedMetrics | None = None + micro: AveragedMetrics | None = None + + +@dataclass(slots=True) +class _ConfusionData: + """Internal: confusion matrix and per-class counts derived from results.""" + + classes: list[str] + matrix: list[list[int]] + n_total: int + n_scored: int + n_skipped: int + oov_fn: list[int] + + +def _build_confusion( + results: list[EvaluationResultDto], + classes: list[str], +) -> _ConfusionData: + """Build a confusion matrix from per-datapoint results. + + Results without a parseable justification are counted in ``n_skipped`` and + omitted. A datapoint whose *predicted* label is out of vocabulary but whose + *expected* label is in vocabulary is a miss for that true class: it is + counted as a false negative (``oov_fn``) rather than dropped, so recall is + not inflated. A datapoint whose *expected* label is out of vocabulary is + skipped — there is no in-vocab true class to attribute the miss to. Labels + are normalized to lowercase for the lookup index so a classifier returning + "Book" vs configured "book" still matches, but the user-supplied casing is + preserved in the returned ``_ConfusionData.classes``. + """ + index_of = {c.lower(): i for i, c in enumerate(classes)} + k = len(classes) + matrix = [[0] * k for _ in range(k)] + oov_fn = [0] * k + + n_total = len(results) + n_scored = 0 + n_skipped = 0 + + for r in results: + j = BaseEvaluatorJustification.try_from(r.details) + if j is None: + n_skipped += 1 + continue + exp = j.expected.lower() + act = j.actual.lower() + if exp not in index_of: + n_skipped += 1 + continue + if act not in index_of: + oov_fn[index_of[exp]] += 1 + n_scored += 1 + continue + matrix[index_of[act]][index_of[exp]] += 1 + n_scored += 1 + + return _ConfusionData( + classes=list(classes), + matrix=matrix, + n_total=n_total, + n_scored=n_scored, + n_skipped=n_skipped, + oov_fn=oov_fn, + ) + + +def _f_beta(precision: float, recall: float, beta: float) -> float: + b2 = beta * beta + # denom == 0 iff precision == recall == 0 (both terms are non-negative and + # beta > 0), which is exactly the zero-score case. + denom = b2 * precision + recall + if denom == 0: + return 0.0 + return (1 + b2) * precision * recall / denom + + +class ClassificationDatasetEvaluator(BaseDatasetEvaluator): + """One implementation for all classification aggregators. + + Scalar variants (precision / recall / fscore) compute the full per-class + P/R/F report and pick the headline by the spec's ``averaging``; the + ``confusion_matrix`` variant returns only the raw grid. + """ + + def evaluate(self, results: list[EvaluationResultDto]) -> EvaluationResult: + """Compute the configured metric report and return the headline as score.""" + confusion = _build_confusion(results, self.classes) + + if isinstance(self.spec, ConfusionMatrixAggregatorSpec): + # No scalar headline — emit the raw grid and let the UI render it. + details = ClassificationDetails( + metric=self.spec.type, + classes=confusion.classes, + confusion_matrix=confusion.matrix, + n_total=confusion.n_total, + n_scored=confusion.n_scored, + n_skipped=confusion.n_skipped, + ) + return NumericEvaluationResult(score=0.0, details=details) + + f_value = ( + self.spec.f_value if isinstance(self.spec, FScoreAggregatorSpec) else 1.0 + ) + k = len(confusion.classes) + + per_class: dict[str, PerClassMetrics] = {} + precisions: list[float] = [] + recalls: list[float] = [] + f_scores: list[float] = [] + total_tp = total_fp = total_fn = 0 + + for c, label in enumerate(confusion.classes): + tp = confusion.matrix[c][c] + row_sum = sum(confusion.matrix[c]) # predicted as `label` + col_sum = sum(confusion.matrix[j][c] for j in range(k)) # true `label` + fp = row_sum - tp + fn = col_sum - tp + confusion.oov_fn[c] + tn = confusion.n_scored - tp - fp - fn + + precision = tp / row_sum if row_sum > 0 else 0.0 + recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0 + f_score = _f_beta(precision, recall, f_value) + + per_class[label] = PerClassMetrics( + tp=tp, + tn=tn, + fp=fp, + fn=fn, + support=tp + fn, + precision=precision, + recall=recall, + f_score=f_score, + ) + precisions.append(precision) + recalls.append(recall) + f_scores.append(f_score) + total_tp += tp + total_fp += fp + total_fn += fn + + # AggregatorSpec classes come from the ExactMatch config which requires + # a non-empty list, so k >= 1 always. + macro = AveragedMetrics( + precision=sum(precisions) / k, + recall=sum(recalls) / k, + f_score=sum(f_scores) / k, + ) + micro_p = total_tp / (total_tp + total_fp) if (total_tp + total_fp) > 0 else 0.0 + micro_r = total_tp / (total_tp + total_fn) if (total_tp + total_fn) > 0 else 0.0 + micro = AveragedMetrics( + precision=micro_p, + recall=micro_r, + f_score=_f_beta(micro_p, micro_r, f_value), + ) + + averaged = micro if self.spec.averaging == "micro" else macro + headline = { + "precision": averaged.precision, + "recall": averaged.recall, + "fscore": averaged.f_score, + }[self.spec.type] + + details = ClassificationDetails( + metric=self.spec.type, + averaging=self.spec.averaging, + f_value=f_value, + classes=confusion.classes, + confusion_matrix=confusion.matrix, + per_class=per_class, + macro=macro, + micro=micro, + n_total=confusion.n_total, + n_scored=confusion.n_scored, + n_skipped=confusion.n_skipped, + ) + return NumericEvaluationResult(score=headline, details=details) diff --git a/packages/uipath/src/uipath/eval/evaluators/contains_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/contains_evaluator.py index c002b47d6..f4f30f7cc 100644 --- a/packages/uipath/src/uipath/eval/evaluators/contains_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/contains_evaluator.py @@ -1,10 +1,10 @@ -"""Contains evaluator for agent outputs.""" +"""Contains evaluator for workload outputs.""" from ..models import ( - AgentExecution, EvaluationResult, EvaluatorType, NumericEvaluationResult, + WorkloadExecution, ) from .base_evaluator import BaseEvaluationCriteria, BaseEvaluatorJustification from .output_evaluator import ( @@ -45,22 +45,22 @@ def get_evaluator_id(cls) -> str: async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: ContainsEvaluationCriteria, ) -> EvaluationResult: """Evaluate whether actual output contains the expected output. Args: - agent_execution: The execution details containing: + workload_execution: The execution details containing: - agent_input: The input received by the agent - - agent_output: The actual output from the agent - - agent_trace: The execution spans to use for the evaluation + - workload_output: The actual output from the agent + - workload_trace: The execution spans to use for the evaluation evaluation_criteria: The criteria to evaluate Returns: EvaluationResult: Boolean result indicating if output contains expected value (True/False) """ - actual_output = str(self._get_actual_output(agent_execution)) + actual_output = str(self._get_actual_output(workload_execution)) expected_output = str(self._get_expected_output(evaluation_criteria)) if not self.evaluator_config.case_sensitive: diff --git a/packages/uipath/src/uipath/eval/evaluators/dataset_evaluator_factory.py b/packages/uipath/src/uipath/eval/evaluators/dataset_evaluator_factory.py new file mode 100644 index 000000000..d30a5748b --- /dev/null +++ b/packages/uipath/src/uipath/eval/evaluators/dataset_evaluator_factory.py @@ -0,0 +1,65 @@ +"""Factory that instantiates dataset-level evaluators from aggregator specs. + +Dataset evaluators are built from a self-contained :class:`AggregatorSpec` +embedded in a per-datapoint classification evaluator's config, plus the source +evaluator's name (supplied by the runtime when walking those configs). All +three aggregator types share a single :class:`ClassificationDatasetEvaluator` +implementation that dispatches on ``spec.type`` internally. +""" + +from __future__ import annotations + +from typing import Sequence + +from ._aggregator_specs import AggregatorSpec +from .classification_dataset_evaluators import ClassificationDatasetEvaluator + + +def build_dataset_evaluator( + spec: AggregatorSpec, + source_evaluator: str, + classes: list[str], +) -> ClassificationDatasetEvaluator: + """Build a dataset evaluator instance from an aggregator spec. + + Args: + spec: A validated :class:`AggregatorSpec` (precision / recall / fscore). + source_evaluator: Name of the per-datapoint evaluator whose results + this aggregator consumes. + classes: The class vocabulary from the parent evaluator's config. Shared + across all aggregators attached to that evaluator — a spec no longer + carries classes of its own. + """ + return ClassificationDatasetEvaluator(spec, source_evaluator, classes) + + +def unique_aggregator_specs(specs: Sequence[AggregatorSpec]) -> list[AggregatorSpec]: + """Drop exact-duplicate specs (same type and parameters), preserving order.""" + seen: set[str] = set() + unique: list[AggregatorSpec] = [] + for spec in specs: + dumped = spec.model_dump_json() + if dumped not in seen: + seen.add(dumped) + unique.append(spec) + return unique + + +def dataset_result_key( + source_evaluator: str, spec: AggregatorSpec, duplicate_type: bool +) -> str: + """Result-map key shared by `uipath eval` and the platform worker. + + ``{source}::{type}``, extended with ``.{averaging}`` (and ``.fb{f_value}`` + for fscore) when the same type appears more than once on one source. + Callers must dedupe via :func:`unique_aggregator_specs` first — after that, + duplicate types always differ in averaging or f_value. + """ + key = f"{source_evaluator}::{spec.type}" + averaging = getattr(spec, "averaging", None) + if not duplicate_type or averaging is None: + return key + f_value = getattr(spec, "f_value", None) + if f_value is not None: + return f"{key}.{averaging}.fb{f_value}" + return f"{key}.{averaging}" diff --git a/packages/uipath/src/uipath/eval/evaluators/exact_match_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/exact_match_evaluator.py index 0f1b3e8e8..2cb7d70f1 100644 --- a/packages/uipath/src/uipath/eval/evaluators/exact_match_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/exact_match_evaluator.py @@ -1,11 +1,14 @@ -"""Exact match evaluator for agent outputs.""" +"""Exact match evaluator for workload outputs.""" + +from pydantic import Field, model_validator from ..models import ( - AgentExecution, EvaluationResult, EvaluatorType, NumericEvaluationResult, + WorkloadExecution, ) +from ._aggregator_specs import AggregatorSpec from .base_evaluator import BaseEvaluatorJustification from .output_evaluator import ( OutputEvaluationCriteria, @@ -20,6 +23,67 @@ class ExactMatchEvaluatorConfig(OutputEvaluatorConfig[OutputEvaluationCriteria]) name: str = "ExactMatchEvaluator" case_sensitive: bool = False negated: bool = False + classes: list[str] | None = Field( + default=None, + description=( + "Label vocabulary shared by every aggregator on this evaluator. " + "Labels are matched case-insensitively against the per-datapoint " + "expected/actual outputs." + ), + ) + aggregators: list[AggregatorSpec] | None = Field( + default=None, + description=( + "Dataset-level metrics (precision / recall / F-score / confusion " + "matrix) computed over the per-datapoint match outcomes. Requires " + "``classes``." + ), + ) + + @model_validator(mode="after") + def _validate_aggregators(self) -> "ExactMatchEvaluatorConfig": + """Aggregators need a usable class vocabulary and per-label outcomes.""" + if not self.aggregators: + return self + if not self.classes: + raise ValueError( + f"ExactMatch evaluator '{self.name}' declares aggregators but no " + "``classes`` list. Set ``classes`` to the label vocabulary the " + "aggregators should compute Precision/Recall/F-score over." + ) + if self.line_by_line_evaluator: + raise ValueError( + f"ExactMatch evaluator '{self.name}': aggregators are not " + "supported with line_by_line_evaluator — per-line results carry " + "no expected/actual labels, so every datapoint would be skipped." + ) + if self.case_sensitive: + raise ValueError( + f"ExactMatch evaluator '{self.name}': aggregators are not " + "supported with case_sensitive — the confusion matrix buckets " + "labels case-insensitively, so a datapoint could score 0.0 yet " + "land on the true-positive diagonal." + ) + if self.negated: + raise ValueError( + f"ExactMatch evaluator '{self.name}': aggregators are not " + "supported with negated — negation flips only the per-datapoint " + "score, not the justification's expected/actual labels, so the " + "confusion matrix would put matches on the true-positive diagonal " + "while they scored 0.0 (and vice versa)." + ) + lowered = [c.lower() for c in self.classes] + if any(not c.strip() or c != c.strip() for c in self.classes) or len( + set(lowered) + ) != len(lowered): + raise ValueError( + f"ExactMatch evaluator '{self.name}': ``classes`` must be " + "non-blank, have no leading/trailing whitespace, and be unique " + "case-insensitively — labels are matched case-insensitively, so " + "duplicates would collapse onto one matrix index, and padded " + "labels would never match anything." + ) + return self class ExactMatchEvaluator( @@ -41,22 +105,22 @@ def get_evaluator_id(cls) -> str: async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: OutputEvaluationCriteria, ) -> EvaluationResult: """Evaluate whether actual output exactly matches expected output. Args: - agent_execution: The execution details containing: + workload_execution: The execution details containing: - agent_input: The input received by the agent - - agent_output: The actual output from the agent - - agent_trace: The execution spans to use for the evaluation + - workload_output: The actual output from the agent + - workload_trace: The execution spans to use for the evaluation evaluation_criteria: The criteria to evaluate Returns: EvaluationResult: Boolean result indicating exact match (True/False) """ - actual_output = self._get_actual_output(agent_execution) + actual_output = self._get_actual_output(workload_execution) expected_output = self._get_expected_output(evaluation_criteria) if isinstance(actual_output, str) or isinstance(expected_output, str): diff --git a/packages/uipath/src/uipath/eval/evaluators/json_similarity_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/json_similarity_evaluator.py index 552194f2e..9ab67be22 100644 --- a/packages/uipath/src/uipath/eval/evaluators/json_similarity_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/json_similarity_evaluator.py @@ -4,10 +4,10 @@ from typing import Any, Tuple from ..models import ( - AgentExecution, EvaluationResult, EvaluatorType, NumericEvaluationResult, + WorkloadExecution, ) from .base_evaluator import BaseEvaluatorJustification from .output_evaluator import ( @@ -51,7 +51,7 @@ def get_evaluator_id(cls) -> str: async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: OutputEvaluationCriteria, ) -> EvaluationResult: """Evaluate similarity between expected and actual JSON outputs. @@ -59,7 +59,7 @@ async def evaluate( Uses token-based comparison with tolerance for numeric differences and Levenshtein distance for string similarity. - agent_execution: The execution details containing: + workload_execution: The execution details containing: - agent_input: The input received by the agent - actual_output: The actual output from the agent - spans: The execution spans to use for the evaluation @@ -69,7 +69,7 @@ async def evaluate( EvaluationResult: Numerical score between 0-100 indicating similarity """ expected_output = self._get_expected_output(evaluation_criteria) - actual_output = self._get_actual_output(agent_execution) + actual_output = self._get_actual_output(workload_execution) score, justification = self._compare_json( expected_output, actual_output, diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_context_precision_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/legacy_context_precision_evaluator.py index 3b0468363..e13776950 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_context_precision_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_context_precision_evaluator.py @@ -11,14 +11,14 @@ from .._execution_context import eval_set_run_id_context from ..models import NumericEvaluationResult -from ..models.models import AgentExecution, EvaluationResult +from ..models.models import EvaluationResult, WorkloadExecution from .base_legacy_evaluator import ( BaseLegacyEvaluator, LegacyEvaluationCriteria, LegacyEvaluatorConfig, track_evaluation_metrics, ) -from .legacy_evaluator_utils import clean_model_name, serialize_object +from .legacy_evaluator_utils import serialize_object class LegacyContextPrecisionEvaluatorConfig(LegacyEvaluatorConfig): @@ -96,7 +96,7 @@ class LegacyContextPrecisionEvaluator( ): """Legacy evaluator that assesses context precision using an LLM. - This evaluator extracts context grounding spans from agent execution traces + This evaluator extracts context grounding spans from workload execution traces and uses an LLM to score the relevance of each chunk to its corresponding query. The final score is the mean of all chunk relevancy scores (normalized to 0-1). """ @@ -125,13 +125,13 @@ def _initialize_llm(self): @track_evaluation_metrics async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: LegacyEvaluationCriteria, ) -> EvaluationResult: - """Evaluate context precision from agent execution traces. + """Evaluate context precision from workload execution traces. Args: - agent_execution: The execution details containing agent_trace with spans + workload_execution: The execution details containing workload_trace with spans evaluation_criteria: Legacy evaluation criteria (unused for context precision) Returns: @@ -143,13 +143,13 @@ async def evaluate( # Extract context grounding spans from the trace context_groundings = self._extract_context_groundings( - agent_execution.agent_trace + workload_execution.workload_trace ) if not context_groundings: return NumericEvaluationResult( score=0.0, - details="No context grounding tool calls found in the agent execution trace.", + details="No context grounding tool calls found in the workload execution trace.", ) # Evaluate each context grounding call @@ -224,16 +224,16 @@ def _parse_span_value(self, value_str: str) -> Any: raise ValueError(f"Cannot parse value: {value_str}") from e def _extract_context_groundings( - self, agent_trace: list[Any] + self, workload_trace: list[Any] ) -> list[dict[str, Any]]: - """Extract context groundings from agent execution trace. + """Extract context groundings from workload execution trace. Looks for spans with input.value and output.value attributes that represent context grounding tool calls. """ context_groundings = [] - for span in agent_trace: + for span in workload_trace: if not hasattr(span, "attributes") or span.attributes is None: continue @@ -326,8 +326,7 @@ async def _get_structured_llm_response( ToolParametersDefinition, ) - # Remove community-agents suffix from llm model name - model = clean_model_name(self.model) + model = self.model # Create tool definition for context precision evaluation # Note: We pass the array schema as a raw dict because ToolPropertyDefinition diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_csv_exact_match_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/legacy_csv_exact_match_evaluator.py index 0b7bac2b4..9441ae65a 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_csv_exact_match_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_csv_exact_match_evaluator.py @@ -12,7 +12,7 @@ EvaluationResult, ) -from ..models.models import AgentExecution +from ..models.models import WorkloadExecution from .base_legacy_evaluator import LegacyEvaluationCriteria, LegacyEvaluatorConfig from .legacy_deterministic_evaluator_base import BaseLegacyDeterministicEvaluator from .line_by_line_utils import wrap_line_in_structure @@ -59,15 +59,15 @@ def validate_target_sub_output_key(cls, v: str) -> str: async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: LegacyEvaluationCriteria, ) -> EvaluationResult: """Evaluate whether specific CSV columns exactly match between actual and expected. Args: - agent_execution: The execution details containing: + workload_execution: The execution details containing: - agent_input: The input received by the agent - - agent_output: The actual output from the agent (can be CSV string or job attachment) + - workload_output: The actual output from the agent (can be CSV string or job attachment) - spans: The execution spans to use for the evaluation evaluation_criteria: The criteria to evaluate containing expected output @@ -78,7 +78,7 @@ async def evaluate( ValueError: If CSV format is invalid or required columns are missing """ # Get actual output (handles job attachments and target_output_key extraction) - actual_output = self._get_actual_output(agent_execution) + actual_output = self._get_actual_output(workload_execution) # Get expected output from criteria expected_output = evaluation_criteria.expected_output @@ -241,7 +241,7 @@ def _do_values_match( async def _evaluate_line_by_line( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: LegacyEvaluationCriteria, ) -> EvaluationResult: """Override line-by-line evaluation to handle CSV structure properly. @@ -255,7 +255,7 @@ async def _evaluate_line_by_line( 5. Aggregates results Args: - agent_execution: The execution details + workload_execution: The execution details evaluation_criteria: The evaluation criteria Returns: @@ -271,7 +271,7 @@ async def _evaluate_line_by_line( ) # Get actual output (this handles job attachments automatically) - actual_output = self._get_actual_output(agent_execution) + actual_output = self._get_actual_output(workload_execution) # Get expected output from criteria expected_output = evaluation_criteria.expected_output @@ -330,18 +330,18 @@ async def _evaluate_line_by_line( else: expected_mini_csv = expected_header # Just header, will fail validation - # Create a modified agent execution for this line - line_agent_execution = AgentExecution( - agent_input=agent_execution.agent_input, - agent_output=wrap_line_in_structure( + # Create a modified workload execution for this line + line_agent_execution = WorkloadExecution( + agent_input=workload_execution.agent_input, + workload_output=wrap_line_in_structure( actual_mini_csv, self.target_output_key ), - agent_trace=agent_execution.agent_trace, + workload_trace=workload_execution.workload_trace, expected_agent_behavior=getattr( - agent_execution, "expected_agent_behavior", None + workload_execution, "expected_agent_behavior", None ), simulation_instructions=getattr( - agent_execution, "simulation_instructions", "" + workload_execution, "simulation_instructions", "" ), ) diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_evaluator_utils.py b/packages/uipath/src/uipath/eval/evaluators/legacy_evaluator_utils.py index b8c20f372..b8ee81906 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_evaluator_utils.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_evaluator_utils.py @@ -3,22 +3,6 @@ import json from typing import Any, Optional -from ..._utils.constants import COMMUNITY_agents_SUFFIX - - -def clean_model_name(model: str) -> str: - """Remove community-agents suffix from model name. - - Args: - model: Model name that may have the community suffix - - Returns: - Model name without the community suffix - """ - if model.endswith(COMMUNITY_agents_SUFFIX): - return model.replace(COMMUNITY_agents_SUFFIX, "") - return model - def serialize_object( content: Any, diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_exact_match_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/legacy_exact_match_evaluator.py index 42ffae047..f20ea2bdf 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_exact_match_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_exact_match_evaluator.py @@ -1,9 +1,9 @@ -"""Exact match evaluator for binary pass/fail evaluation of agent outputs.""" +"""Exact match evaluator for binary pass/fail evaluation of workload outputs.""" from uipath.eval.models import BooleanEvaluationResult, EvaluationResult from .._helpers.output_path import resolve_output_path -from ..models.models import AgentExecution +from ..models.models import WorkloadExecution from .base_legacy_evaluator import LegacyEvaluationCriteria, LegacyEvaluatorConfig from .legacy_deterministic_evaluator_base import BaseLegacyDeterministicEvaluator @@ -26,13 +26,13 @@ class LegacyExactMatchEvaluator( async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: LegacyEvaluationCriteria, ) -> EvaluationResult: """Evaluate whether actual output exactly matches expected output. Args: - agent_execution: The execution details containing: + workload_execution: The execution details containing: - agent_input: The input received by the agent - actual_output: The actual output from the agent - spans: The execution spans to use for the evaluation @@ -41,7 +41,7 @@ async def evaluate( Returns: EvaluationResult: Boolean result indicating exact match (True/False) """ - actual_output = agent_execution.agent_output + actual_output = workload_execution.workload_output expected_output = evaluation_criteria.expected_output if self.target_output_key and self.target_output_key != "*": diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_faithfulness_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/legacy_faithfulness_evaluator.py index c4eba1fcf..b17ec7f9f 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_faithfulness_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_faithfulness_evaluator.py @@ -1,4 +1,4 @@ -"""Legacy Faithfulness evaluator for assessing whether agent output claims are grounded in context.""" +"""Legacy Faithfulness evaluator for assessing whether workload output claims are grounded in context.""" import json from typing import Any, Optional @@ -10,7 +10,7 @@ from .._execution_context import eval_set_run_id_context from ..models import NumericEvaluationResult -from ..models.models import AgentExecution, EvaluationResult +from ..models.models import EvaluationResult, WorkloadExecution from .base_legacy_evaluator import ( BaseLegacyEvaluator, LegacyEvaluationCriteria, @@ -18,7 +18,6 @@ track_evaluation_metrics, ) from .legacy_evaluator_utils import ( - clean_model_name, serialize_object, ) @@ -35,9 +34,9 @@ class LegacyFaithfulnessEvaluator( ): """Legacy evaluator that assesses faithfulness using an LLM. - This evaluator extracts claims from agent output using a 3-stage pipeline + This evaluator extracts claims from workload output using a 3-stage pipeline (selection, disambiguation, decomposition) and evaluates whether each claim - is grounded in the available context sources extracted from agent traces. + is grounded in the available context sources extracted from workload traces. The final score is the percentage of claims that are grounded. """ @@ -63,13 +62,13 @@ def _initialize_llm(self): @track_evaluation_metrics async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: LegacyEvaluationCriteria, ) -> EvaluationResult: - """Evaluate faithfulness of agent output against available context. + """Evaluate faithfulness of workload output against available context. Args: - agent_execution: The execution details containing agent_trace with spans + workload_execution: The execution details containing workload_trace with spans evaluation_criteria: Legacy evaluation criteria containing expected_output Returns: @@ -79,30 +78,32 @@ async def evaluate( if self.llm is None: self._initialize_llm() - # Extract agent output - agent_output = str(evaluation_criteria.expected_output or "") - if not agent_output or not agent_output.strip(): + # Extract workload output + workload_output = str(evaluation_criteria.expected_output or "") + if not workload_output or not workload_output.strip(): return NumericEvaluationResult( score=0.0, - details="No agent output provided for faithfulness evaluation.", + details="No workload output provided for faithfulness evaluation.", ) # Extract context sources from traces - context_sources = self._extract_context_sources(agent_execution.agent_trace) + context_sources = self._extract_context_sources( + workload_execution.workload_trace + ) if not context_sources: return NumericEvaluationResult( score=0.0, - details="No context sources found in the agent execution trace.", + details="No context sources found in the workload execution trace.", ) - # Stage 1: Extract verifiable claims from agent output - claims = await self._extract_claims(agent_output) + # Stage 1: Extract verifiable claims from workload output + claims = await self._extract_claims(workload_output) if not claims: return NumericEvaluationResult( score=100.0, - details="No verifiable claims found in agent output.", + details="No verifiable claims found in workload output.", ) # Stage 2: Evaluate each claim against context sources @@ -127,8 +128,10 @@ async def evaluate( details=justification, ) - def _extract_context_sources(self, agent_trace: list[Any]) -> list[dict[str, str]]: - """Extract context sources from agent execution trace. + def _extract_context_sources( + self, workload_trace: list[Any] + ) -> list[dict[str, str]]: + """Extract context sources from workload execution trace. Looks for tool call outputs and context grounding spans that provide context. @@ -137,7 +140,7 @@ def _extract_context_sources(self, agent_trace: list[Any]) -> list[dict[str, str """ context_sources = [] - for span in agent_trace: + for span in workload_trace: if not hasattr(span, "attributes") or span.attributes is None: continue @@ -180,8 +183,8 @@ def _serialize_content(self, content: Any) -> str: """Serialize content to string format.""" return serialize_object(content, sort_keys=False) - async def _extract_claims(self, agent_output: str) -> list[dict[str, str]]: - """Extract verifiable claims from agent output using 3-stage pipeline. + async def _extract_claims(self, workload_output: str) -> list[dict[str, str]]: + """Extract verifiable claims from workload output using 3-stage pipeline. Stages: 1. Selection: Filter to verifiable sentences @@ -192,23 +195,25 @@ async def _extract_claims(self, agent_output: str) -> list[dict[str, str]]: List of claim dicts with 'text' and 'original_sentence' keys """ # Stage 1: Selection - verifiable_sentences = await self._select_verifiable_sentences(agent_output) + verifiable_sentences = await self._select_verifiable_sentences(workload_output) if not verifiable_sentences: return [] # Stage 2: Disambiguation disambiguated_sentences = await self._disambiguate_sentences( - verifiable_sentences, agent_output + verifiable_sentences, workload_output ) if not disambiguated_sentences: return [] # Stage 3: Decomposition - claims = await self._decompose_to_claims(disambiguated_sentences, agent_output) + claims = await self._decompose_to_claims( + disambiguated_sentences, workload_output + ) return claims - async def _select_verifiable_sentences(self, agent_output: str) -> list[str]: - """Stage 1: Filter agent output to verifiable sentences.""" + async def _select_verifiable_sentences(self, workload_output: str) -> list[str]: + """Stage 1: Filter workload output to verifiable sentences.""" prompt = f"""You are an expert evaluator identifying verifiable claims. TASK: Identify sentences in the agent output that contain verifiable, factual claims. @@ -217,9 +222,9 @@ async def _select_verifiable_sentences(self, agent_output: str) -> list[str]: OUTPUT FORMAT: Return a JSON object with a "sentences" field containing an array of strings. Each string should be a complete sentence from the original output. - -{agent_output} - + +{workload_output} + Identify and return only the verifiable sentences.""" @@ -507,8 +512,7 @@ async def _get_structured_llm_response( ToolParametersDefinition, ) - # Remove community-agents suffix from llm model name - model = clean_model_name(self.model) + model = self.model # Create a dynamic tool definition based on the schema tool = ToolDefinition( diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_json_similarity_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/legacy_json_similarity_evaluator.py index 70fe28b2e..0fbb99b8e 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_json_similarity_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_json_similarity_evaluator.py @@ -5,7 +5,7 @@ from .._helpers.output_path import resolve_output_path from ..models import EvaluationResult, NumericEvaluationResult -from ..models.models import AgentExecution +from ..models.models import WorkloadExecution from .base_legacy_evaluator import LegacyEvaluationCriteria, LegacyEvaluatorConfig from .legacy_deterministic_evaluator_base import BaseLegacyDeterministicEvaluator @@ -30,7 +30,7 @@ class LegacyJsonSimilarityEvaluator( async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: LegacyEvaluationCriteria, ) -> EvaluationResult: """Evaluate similarity between expected and actual JSON outputs. @@ -38,7 +38,7 @@ async def evaluate( Uses token-based comparison with tolerance for numeric differences and Levenshtein distance for string similarity. - agent_execution: The execution details containing: + workload_execution: The execution details containing: - agent_input: The input received by the agent - actual_output: The actual output from the agent - spans: The execution spans to use for the evaluation @@ -47,7 +47,7 @@ async def evaluate( Returns: EvaluationResult: Numerical score between 0-100 indicating similarity """ - actual_output = agent_execution.agent_output + actual_output = workload_execution.workload_output expected_output = evaluation_criteria.expected_output if self.target_output_key and self.target_output_key != "*": diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_llm_as_judge_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/legacy_llm_as_judge_evaluator.py index 0c676ac9b..ef2ab8a34 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_llm_as_judge_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_llm_as_judge_evaluator.py @@ -1,4 +1,4 @@ -"""LLM-as-a-judge evaluator for subjective quality assessment of agent outputs.""" +"""LLM-as-a-judge evaluator for subjective quality assessment of workload outputs.""" import logging from typing import Any, Optional @@ -9,17 +9,16 @@ from uipath.platform.chat import UiPathLlmChatService from uipath.platform.chat.llm_gateway import RequiredToolChoice -from ..._utils.constants import COMMUNITY_agents_SUFFIX from .._execution_context import eval_set_run_id_context from .._helpers.helpers import is_empty_value from .._helpers.output_path import resolve_output_path from ..models import NumericEvaluationResult from ..models.models import ( - AgentExecution, EvaluationResult, LLMResponse, UiPathEvaluationError, UiPathEvaluationErrorCategory, + WorkloadExecution, ) from .base_legacy_evaluator import ( BaseLegacyEvaluator, @@ -38,7 +37,7 @@ class LegacyLlmAsAJudgeEvaluatorConfig(LegacyEvaluatorConfig): class LegacyLlmAsAJudgeEvaluator(BaseLegacyEvaluator[LegacyLlmAsAJudgeEvaluatorConfig]): - """Legacy evaluator that uses an LLM to judge the quality of agent output.""" + """Legacy evaluator that uses an LLM to judge the quality of workload output.""" prompt: str model: str = Field(default="same-as-agent") @@ -104,7 +103,7 @@ def _initialize_llm(self): async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: LegacyEvaluationCriteria, ) -> EvaluationResult: """Evaluate using an LLM as a judge. @@ -112,7 +111,7 @@ async def evaluate( Sends the formatted prompt to the configured LLM and expects a JSON response with a numerical score (0-100) and justification. - agent_execution: The execution details containing: + workload_execution: The execution details containing: - agent_input: The input received by the agent - actual_output: The actual output from the agent - spans: The execution spans to use for the evaluation @@ -125,7 +124,7 @@ async def evaluate( if self.llm is None: self._initialize_llm() - actual_output = agent_execution.agent_output + actual_output = workload_execution.workload_output expected_output = evaluation_criteria.expected_output if self.target_output_key and self.target_output_key != "*": @@ -193,10 +192,7 @@ async def _get_llm_response(self, evaluation_prompt: str) -> LLMResponse: Returns: LLMResponse with score and justification """ - # remove community-agents suffix from llm model name model = self.model - if model.endswith(COMMUNITY_agents_SUFFIX): - model = model.replace(COMMUNITY_agents_SUFFIX, "") # Create evaluation tool for function calling (works across all models) evaluation_tool = create_evaluation_tool() diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_llm_helpers.py b/packages/uipath/src/uipath/eval/evaluators/legacy_llm_helpers.py index 67aaee0c3..8cb8da0de 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_llm_helpers.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_llm_helpers.py @@ -1,6 +1,7 @@ """Helper functions for legacy LLM evaluators using function calling.""" import logging +import math from typing import Any from uipath.platform.chat.llm_gateway import ( @@ -89,9 +90,32 @@ def extract_tool_call_response(response: Any, model: str) -> LLMResponse: logger.debug(f"Arguments: {arguments}") raise ValueError(error_msg) - score = float(arguments["score"]) + try: + score = float(arguments["score"]) + except (ValueError, TypeError) as e: + error_msg = ( + f"Non-numeric score {arguments['score']!r} in tool call arguments " + f"from model {model}: expected a number between 0 and 100" + ) + logger.error(f"❌ {error_msg}") + logger.debug(f"Arguments: {arguments}") + raise ValueError(error_msg) from e + justification = str(arguments["justification"]) + # Models occasionally emit corrupted numeric tool arguments despite the + # 0-100 range stated in the tool schema (e.g. gemini-2.5-flash returning + # 989898 or 950). Unvalidated, such a value poisons every run-level + # aggregate downstream, so reject it and let the evaluation surface as + # an error instead of recording a fabricated score. + if not math.isfinite(score) or score < 0.0 or score > 100.0: + error_msg = ( + f"Invalid score {score!r} in tool call arguments from model {model}: " + f"expected a number between 0 and 100" + ) + logger.error(f"❌ {error_msg}") + raise ValueError(error_msg) + logger.debug( f"✅ Extracted score: {score}, justification length: {len(justification)} chars" ) diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py index 17b69d0d0..1368d9339 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py @@ -10,17 +10,16 @@ from uipath.platform.chat import UiPathLlmChatService from uipath.platform.chat.llm_gateway import RequiredToolChoice -from ..._utils.constants import COMMUNITY_agents_SUFFIX from .._execution_context import eval_set_run_id_context +from .._helpers.evaluators_helpers import trace_to_str from .._helpers.helpers import is_empty_value from ..models import EvaluationResult from ..models.models import ( - AgentExecution, LLMResponse, NumericEvaluationResult, - TrajectoryEvaluationTrace, UiPathEvaluationError, UiPathEvaluationErrorCategory, + WorkloadExecution, ) from .base_legacy_evaluator import ( BaseLegacyEvaluator, @@ -75,7 +74,7 @@ def _initialize_llm(self): async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: LegacyEvaluationCriteria, ) -> EvaluationResult: """Evaluate using trajectory analysis. @@ -83,10 +82,10 @@ async def evaluate( Analyzes the execution path and decision sequence taken by the agent. Args: - agent_execution: The execution details containing: + workload_execution: The execution details containing: - agent_input: The input received by the agent - actual_output: The actual output from the agent - - agent_trace: The execution spans to use for the evaluation + - workload_trace: The execution spans to use for the evaluation - expected_agent_behavior: The expected agent behavior evaluation_criteria: The criteria to evaluate Returns: @@ -100,8 +99,8 @@ async def evaluate( self._initialize_llm() evaluation_prompt = self._create_evaluation_prompt( - expected_agent_behavior=agent_execution.expected_agent_behavior, - agent_run_history=agent_execution.agent_trace, + expected_agent_behavior=workload_execution.expected_agent_behavior, + agent_run_history=workload_execution.workload_trace, ) llm_response = await self._get_llm_response(evaluation_prompt) @@ -140,10 +139,7 @@ def _create_evaluation_prompt( and agent_run_history and isinstance(agent_run_history[0], ReadableSpan) ): - trajectory_trace = TrajectoryEvaluationTrace.from_readable_spans( - agent_run_history - ) - agent_run_history = str(trajectory_trace.spans) + agent_run_history = trace_to_str(agent_run_history) else: agent_run_history = str(agent_run_history) @@ -166,8 +162,6 @@ async def _get_llm_response(self, evaluation_prompt: str) -> LLMResponse: assert self.llm, "LLM should be initialized before calling this method." model = self.model - if model.endswith(COMMUNITY_agents_SUFFIX): - model = model.replace(COMMUNITY_agents_SUFFIX, "") # Create evaluation tool for function calling (works across all models) evaluation_tool = create_evaluation_tool() diff --git a/packages/uipath/src/uipath/eval/evaluators/line_by_line_utils.py b/packages/uipath/src/uipath/eval/evaluators/line_by_line_utils.py index 19182622c..ca1d7d891 100644 --- a/packages/uipath/src/uipath/eval/evaluators/line_by_line_utils.py +++ b/packages/uipath/src/uipath/eval/evaluators/line_by_line_utils.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Any, Callable if TYPE_CHECKING: - from ..models import AgentExecution, EvaluationResult + from ..models import EvaluationResult, WorkloadExecution def split_into_lines( @@ -79,7 +79,7 @@ async def evaluate_lines( actual_lines: list[str], expected_lines: list[str], target_output_key: str, - agent_execution: "AgentExecution", + workload_execution: "WorkloadExecution", evaluate_fn: Callable[[Any, Any], Any], create_line_criteria_fn: Callable[[str], Any], ) -> tuple[list[Any], list[tuple[int, "EvaluationResult"]]]: @@ -89,7 +89,7 @@ async def evaluate_lines( actual_lines: List of actual output lines expected_lines: List of expected output lines target_output_key: Key for wrapping lines - agent_execution: Original agent execution + workload_execution: Original workload execution evaluate_fn: Function to evaluate (line_execution, line_criteria) -> result create_line_criteria_fn: Function to create criteria for a line (expected_line) -> criteria @@ -110,18 +110,18 @@ async def evaluate_lines( # Wrap lines in the same structure as original output line_agent_output = wrap_line_in_structure(actual_line, target_output_key) - # Create a modified agent execution for this line - from ..models.models import AgentExecution + # Create a modified workload execution for this line + from ..models.models import WorkloadExecution - line_agent_execution = AgentExecution( - agent_input=agent_execution.agent_input, - agent_output=line_agent_output, - agent_trace=agent_execution.agent_trace, + line_agent_execution = WorkloadExecution( + agent_input=workload_execution.agent_input, + workload_output=line_agent_output, + workload_trace=workload_execution.workload_trace, expected_agent_behavior=getattr( - agent_execution, "expected_agent_behavior", None + workload_execution, "expected_agent_behavior", None ), simulation_instructions=getattr( - agent_execution, "simulation_instructions", "" + workload_execution, "simulation_instructions", "" ), ) diff --git a/packages/uipath/src/uipath/eval/evaluators/llm_as_judge_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/llm_as_judge_evaluator.py index c67212548..a38bbde64 100644 --- a/packages/uipath/src/uipath/eval/evaluators/llm_as_judge_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/llm_as_judge_evaluator.py @@ -1,4 +1,4 @@ -"""LLM-as-a-judge evaluator for subjective quality assessment of agent outputs.""" +"""LLM-as-a-judge evaluator for subjective quality assessment of workload outputs.""" import copy import json @@ -13,12 +13,11 @@ from uipath.platform.chat import UiPathLlmChatService from .._execution_context import eval_set_run_id_context -from .._helpers.evaluators_helpers import COMMUNITY_agents_SUFFIX from ..models import ( - AgentExecution, EvaluationResult, LLMResponse, NumericEvaluationResult, + WorkloadExecution, ) from ..models.llm_judge_types import ( LLMJudgeOutputSchema, @@ -142,8 +141,8 @@ def _get_llm_service(self): ) from e @abstractmethod - def _get_actual_output(self, agent_execution: AgentExecution) -> Any: - """Get the actual output from the agent execution. Must be implemented by concrete evaluator classes.""" + def _get_actual_output(self, workload_execution: WorkloadExecution) -> Any: + """Get the actual output from the workload execution. Must be implemented by concrete evaluator classes.""" pass @abstractmethod @@ -153,12 +152,12 @@ def _get_expected_output(self, evaluation_criteria: T) -> Any: async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: T, ) -> EvaluationResult: """Evaluate using an LLM as a judge.""" evaluation_prompt = self._create_evaluation_prompt( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria=evaluation_criteria, ) @@ -166,7 +165,7 @@ async def evaluate( validated_justification = self.validate_justification( { "expected": str(self._get_expected_output(evaluation_criteria)), - "actual": str(self._get_actual_output(agent_execution)), + "actual": str(self._get_actual_output(workload_execution)), "justification": llm_response.justification, } ) @@ -178,7 +177,7 @@ async def evaluate( def _create_evaluation_prompt( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: T, ) -> str: """Create the evaluation prompt for the LLM.""" @@ -186,7 +185,7 @@ def _create_evaluation_prompt( formatted_prompt = self.evaluator_config.prompt.replace( self.actual_output_placeholder, - str(self._get_actual_output(agent_execution)), + str(self._get_actual_output(workload_execution)), ) formatted_prompt = formatted_prompt.replace( self.expected_output_placeholder, @@ -211,10 +210,7 @@ async def _get_llm_response(self, evaluation_prompt: str) -> LLMResponse: ToolPropertyDefinition, ) - # Remove community-agents suffix from llm model name model = self.evaluator_config.model - if model.endswith(COMMUNITY_agents_SUFFIX): - model = model.replace(COMMUNITY_agents_SUFFIX, "") # Define function/tool for structured output (works for ALL models via Normalized API) evaluation_tool = ToolDefinition( diff --git a/packages/uipath/src/uipath/eval/evaluators/llm_judge_output_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/llm_judge_output_evaluator.py index d533de237..bd86d229b 100644 --- a/packages/uipath/src/uipath/eval/evaluators/llm_judge_output_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/llm_judge_output_evaluator.py @@ -1,10 +1,10 @@ -"""LLM judge output evaluators for evaluating agent outputs.""" +"""LLM judge output evaluators for evaluating workload outputs.""" from typing import TypeVar from pydantic import BaseModel, Field -from ..models import AgentExecution, EvaluationResult, EvaluatorType +from ..models import EvaluationResult, EvaluatorType, WorkloadExecution from ..models.llm_judge_types import ( LLMJudgeOutputSchema, LLMJudgePromptTemplates, @@ -68,16 +68,18 @@ def get_evaluator_id(cls) -> str: async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: OutputEvaluationCriteria, ) -> EvaluationResult: """Evaluate using an LLM as a judge.""" # Explicitly delegate to LLMJudgeMixin's evaluate method to override BaseEvaluator - return await LLMJudgeMixin.evaluate(self, agent_execution, evaluation_criteria) + return await LLMJudgeMixin.evaluate( + self, workload_execution, evaluation_criteria + ) class LLMJudgeOutputEvaluator(BaseLLMOutputEvaluator[LLMJudgeOutputEvaluatorConfig]): - """Evaluator that uses an LLM to judge the quality of agent output. + """Evaluator that uses an LLM to judge the quality of workload output. Inherits all functionality from BaseLLMOutputEvaluator but uses the standard system prompt and output schema for general output evaluation. @@ -95,7 +97,7 @@ def get_evaluator_id(cls) -> str: class LLMJudgeStrictJSONSimilarityOutputEvaluator( BaseLLMOutputEvaluator[LLMJudgeStrictJSONSimilarityOutputEvaluatorConfig] ): - """Evaluator that uses an LLM to judge the quality of agent output with strict JSON similarity. + """Evaluator that uses an LLM to judge the quality of workload output with strict JSON similarity. Inherits all functionality from BaseLLMOutputEvaluator but uses a different system prompt and output schema specific to strict JSON similarity evaluation. diff --git a/packages/uipath/src/uipath/eval/evaluators/llm_judge_trajectory_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/llm_judge_trajectory_evaluator.py index 69b14cd1f..d8b16ea90 100644 --- a/packages/uipath/src/uipath/eval/evaluators/llm_judge_trajectory_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/llm_judge_trajectory_evaluator.py @@ -1,4 +1,4 @@ -"""LLM judge trajectory evaluator for evaluating agent execution trajectories.""" +"""LLM judge trajectory evaluator for evaluating workload execution trajectories.""" from typing import Any, TypeVar @@ -6,9 +6,9 @@ from .._helpers.evaluators_helpers import trace_to_str from ..models import ( - AgentExecution, EvaluationResult, EvaluatorType, + WorkloadExecution, ) from ..models.llm_judge_types import ( LLMJudgePromptTemplates, @@ -72,15 +72,15 @@ def get_evaluator_id(cls) -> str: async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: TrajectoryEvaluationCriteria, ) -> EvaluationResult: """Evaluate using trajectory analysis.""" - return await super().evaluate(agent_execution, evaluation_criteria) + return await super().evaluate(workload_execution, evaluation_criteria) - def _get_actual_output(self, agent_execution: AgentExecution) -> Any: - """Get the actual output from the agent execution.""" - return trace_to_str(agent_execution.agent_trace) + def _get_actual_output(self, workload_execution: WorkloadExecution) -> Any: + """Get the actual output from the workload execution.""" + return trace_to_str(workload_execution.workload_trace) def _get_expected_output( self, evaluation_criteria: TrajectoryEvaluationCriteria @@ -90,20 +90,20 @@ def _get_expected_output( def _create_evaluation_prompt( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: TrajectoryEvaluationCriteria, ) -> str: """Create the evaluation prompt for the LLM.""" formatted_prompt = super()._create_evaluation_prompt( - agent_execution, evaluation_criteria + workload_execution, evaluation_criteria ) formatted_prompt = formatted_prompt.replace( self.user_input_placeholder, - str(agent_execution.agent_input), + str(workload_execution.agent_input), ) formatted_prompt = formatted_prompt.replace( self.simulation_instructions_placeholder, - agent_execution.simulation_instructions, + workload_execution.simulation_instructions, ) return formatted_prompt diff --git a/packages/uipath/src/uipath/eval/evaluators/multiclass_classification_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/multiclass_classification_evaluator.py index 69790c3aa..ef1ffe0b6 100644 --- a/packages/uipath/src/uipath/eval/evaluators/multiclass_classification_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/multiclass_classification_evaluator.py @@ -1,4 +1,4 @@ -"""Multiclass classification evaluator for agent outputs. +"""Multiclass classification evaluator for workload outputs. Evaluates multiclass classification by comparing predicted vs expected class. Per-datapoint score is 1.0 (correct) or 0.0 (incorrect). The reduce_scores @@ -10,10 +10,10 @@ from typing import Literal from ..models import ( - AgentExecution, EvaluationResult, EvaluatorType, NumericEvaluationResult, + WorkloadExecution, ) from ..models.models import ( EvaluationResultDto, @@ -66,11 +66,11 @@ def get_evaluator_id(cls) -> str: async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: MulticlassClassificationEvaluationCriteria, ) -> EvaluationResult: """Evaluate multiclass classification by comparing predicted vs expected class.""" - predicted_class = str(self._get_actual_output(agent_execution)).lower() + predicted_class = str(self._get_actual_output(workload_execution)).lower() expected_class = evaluation_criteria.expected_class.lower() classes = [c.lower() for c in self.evaluator_config.classes] diff --git a/packages/uipath/src/uipath/eval/evaluators/output_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/output_evaluator.py index 27dd05d9f..d8b99c9c0 100644 --- a/packages/uipath/src/uipath/eval/evaluators/output_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/output_evaluator.py @@ -7,7 +7,7 @@ from pydantic import BaseModel, Field from .._helpers.output_path import resolve_output_path -from ..models import AgentExecution +from ..models import WorkloadExecution from ..models.models import UiPathEvaluationError, UiPathEvaluationErrorCategory from .attachment_utils import ( download_attachment_as_string, @@ -86,8 +86,9 @@ class OutputEvaluatorConfig(BaseEvaluatorConfig[T]): specific output evaluation criteria types while maintaining type safety. """ - target_output_key: str = Field( - default="*", description="Key to extract output from agent execution" + target_output_key: str | list[str] = Field( + default="*", + description="Key or list of keys to extract output from workload execution", ) line_by_line_evaluator: bool = Field( default=False, @@ -127,18 +128,42 @@ def _normalize_numbers(self, obj: Any) -> Any: return float(obj) return obj - def _get_actual_output(self, agent_execution: AgentExecution) -> Any: - """Get the actual output from the agent execution. + def _get_actual_output(self, workload_execution: WorkloadExecution) -> Any: + """Get the actual output from the workload execution. If the output is a job attachment URI, downloads the attachment and returns its content as a string. """ - if self.evaluator_config.target_output_key != "*": - try: - result = resolve_output_path( - agent_execution.agent_output, - self.evaluator_config.target_output_key, + key = self.evaluator_config.target_output_key + + if isinstance(key, list): + if not isinstance(workload_execution.workload_output, dict): + raise UiPathEvaluationError( + code="INVALID_ACTUAL_OUTPUT", + title="When target output keys are specified, actual output must be a dictionary", + detail=f"Got {type(workload_execution.workload_output).__name__}", + category=UiPathEvaluationErrorCategory.USER, ) + try: + list_result: dict[str, Any] = { + k: resolve_output_path(workload_execution.workload_output, k) + for k in key + } + except (KeyError, IndexError, TypeError) as e: + raise UiPathEvaluationError( + code="TARGET_OUTPUT_KEY_NOT_FOUND", + title="One or more target output keys not found in actual output", + detail=f"Error: {e}", + category=UiPathEvaluationErrorCategory.USER, + ) from e + for k, v in list_result.items(): + if is_job_attachment_uri(v): + attachment_id = extract_attachment_id(v) + list_result[k] = download_attachment_as_string(attachment_id) + return self._normalize_numbers(list_result) + elif key != "*": + try: + result = resolve_output_path(workload_execution.workload_output, key) except (KeyError, IndexError, TypeError) as e: raise UiPathEvaluationError( code="TARGET_OUTPUT_KEY_NOT_FOUND", @@ -147,9 +172,8 @@ def _get_actual_output(self, agent_execution: AgentExecution) -> Any: category=UiPathEvaluationErrorCategory.USER, ) from e else: - result = agent_execution.agent_output + result = workload_execution.workload_output - # Check if result is a job attachment URI and download if so if is_job_attachment_uri(result): attachment_id = extract_attachment_id(result) result = download_attachment_as_string(attachment_id) @@ -165,46 +189,81 @@ def _get_full_expected_output(self, evaluation_criteria: T) -> Any: category=UiPathEvaluationErrorCategory.SYSTEM, ) - def _get_expected_output(self, evaluation_criteria: T) -> Any: - """Load the expected output from the evaluation criteria.""" - expected_output = self._get_full_expected_output(evaluation_criteria) - if self.evaluator_config.target_output_key != "*": - if isinstance(expected_output, str): - try: - expected_output = json.loads(expected_output) - except json.JSONDecodeError as e: - raise UiPathEvaluationError( - code="INVALID_EXPECTED_OUTPUT", - title="When target output key is not '*', expected output must be a dictionary or a valid JSON string", - detail=f"Error: {e}", - category=UiPathEvaluationErrorCategory.USER, - ) from e + def _resolve_list_key_expected( + self, expected_output: Any, keys: list[str] + ) -> dict[str, Any]: + """Parse and resolve expected output for a list of keys.""" + if isinstance(expected_output, str): try: - expected_output = resolve_output_path( - expected_output, - self.evaluator_config.target_output_key, - ) - except (KeyError, IndexError, TypeError) as e: + expected_output = json.loads(expected_output) + except json.JSONDecodeError as e: raise UiPathEvaluationError( - code="TARGET_OUTPUT_KEY_NOT_FOUND", - title="Target output key not found in expected output", + code="INVALID_EXPECTED_OUTPUT", + title="When target output keys are specified, expected output must be a dictionary or a valid JSON string", + detail=f"Error: {e}", + category=UiPathEvaluationErrorCategory.USER, + ) from e + if not isinstance(expected_output, dict): + raise UiPathEvaluationError( + code="INVALID_EXPECTED_OUTPUT", + title="When target output keys are specified, expected output must be a dictionary", + detail=f"Got {type(expected_output).__name__}", + category=UiPathEvaluationErrorCategory.USER, + ) + try: + return {k: resolve_output_path(expected_output, k) for k in keys} + except (KeyError, IndexError, TypeError) as e: + raise UiPathEvaluationError( + code="TARGET_OUTPUT_KEY_NOT_FOUND", + title="One or more target output keys not found in expected output", + detail=f"Error: {e}", + category=UiPathEvaluationErrorCategory.USER, + ) from e + + def _resolve_scalar_key_expected(self, expected_output: Any, key: str) -> Any: + """Parse and resolve expected output for a single key.""" + if isinstance(expected_output, str): + try: + expected_output = json.loads(expected_output) + except json.JSONDecodeError as e: + raise UiPathEvaluationError( + code="INVALID_EXPECTED_OUTPUT", + title="When target output key is not '*', expected output must be a dictionary or a valid JSON string", detail=f"Error: {e}", category=UiPathEvaluationErrorCategory.USER, ) from e + try: + return resolve_output_path(expected_output, key) + except (KeyError, IndexError, TypeError) as e: + raise UiPathEvaluationError( + code="TARGET_OUTPUT_KEY_NOT_FOUND", + title="Target output key not found in expected output", + detail=f"Error: {e}", + category=UiPathEvaluationErrorCategory.USER, + ) from e + + def _get_expected_output(self, evaluation_criteria: T) -> Any: + """Load the expected output from the evaluation criteria.""" + expected_output = self._get_full_expected_output(evaluation_criteria) + key = self.evaluator_config.target_output_key + if isinstance(key, list): + expected_output = self._resolve_list_key_expected(expected_output, key) + elif key != "*": + expected_output = self._resolve_scalar_key_expected(expected_output, key) return self._normalize_numbers(expected_output) async def validate_and_evaluate_criteria( self, - agent_execution: "AgentExecution", + workload_execution: "WorkloadExecution", evaluation_criteria: Any, ) -> "EvaluationResult": - """Validate evaluation criteria and evaluate the agent execution. + """Validate evaluation criteria and evaluate the workload execution. If line_by_line_evaluator is enabled, splits the output by delimiter and evaluates each line separately, then aggregates the scores. Args: - agent_execution: The agent execution to evaluate + workload_execution: The workload execution to evaluate evaluation_criteria: The evaluation criteria (dict or typed object) Returns: @@ -225,22 +284,24 @@ async def validate_and_evaluate_criteria( validated_criteria = self.validate_evaluation_criteria(evaluation_criteria) # Check if line-by-line evaluation is enabled - if not self.evaluator_config.line_by_line_evaluator: + if not self.evaluator_config.line_by_line_evaluator or isinstance( + self.evaluator_config.target_output_key, list + ): # Standard evaluation - return await self.evaluate(agent_execution, validated_criteria) + return await self.evaluate(workload_execution, validated_criteria) # Line-by-line evaluation - return await self._evaluate_line_by_line(agent_execution, validated_criteria) + return await self._evaluate_line_by_line(workload_execution, validated_criteria) async def _evaluate_line_by_line( self, - agent_execution: "AgentExecution", + workload_execution: "WorkloadExecution", evaluation_criteria: T, ) -> "EvaluationResult": """Evaluate output line by line and aggregate scores. Args: - agent_execution: The agent execution to evaluate + workload_execution: The workload execution to evaluate evaluation_criteria: Validated evaluation criteria Returns: @@ -248,51 +309,44 @@ async def _evaluate_line_by_line( """ from .line_by_line_utils import build_line_by_line_result, evaluate_lines - # Get the full actual and expected outputs before splitting - actual_output = self._get_actual_output(agent_execution) + key_str = ( + self.evaluator_config.target_output_key + if isinstance(self.evaluator_config.target_output_key, str) + else "*" + ) + + actual_output = self._get_actual_output(workload_execution) expected_output = self._get_expected_output(evaluation_criteria) - # Split into lines using utility function actual_lines = split_into_lines( - actual_output, - self.evaluator_config.line_delimiter, - self.evaluator_config.target_output_key, + actual_output, self.evaluator_config.line_delimiter, key_str ) expected_lines = split_into_lines( - expected_output, - self.evaluator_config.line_delimiter, - self.evaluator_config.target_output_key, + expected_output, self.evaluator_config.line_delimiter, key_str ) - # Store original agent execution data - original_agent_output = agent_execution.agent_output + original_agent_output = workload_execution.workload_output - # Create function to build line criteria def create_line_criteria(expected_line: str) -> Any: from .line_by_line_utils import wrap_line_in_structure - line_expected_output = wrap_line_in_structure( - expected_line, self.evaluator_config.target_output_key - ) + line_expected_output = wrap_line_in_structure(expected_line, key_str) line_criteria_dict = evaluation_criteria.model_dump() if "expected_output" in line_criteria_dict: line_criteria_dict["expected_output"] = line_expected_output return type(evaluation_criteria).model_validate(line_criteria_dict) - # Evaluate all lines using utility function line_details, line_results = await evaluate_lines( actual_lines=actual_lines, expected_lines=expected_lines, - target_output_key=self.evaluator_config.target_output_key, - agent_execution=agent_execution, + target_output_key=key_str, + workload_execution=workload_execution, evaluate_fn=self.evaluate, create_line_criteria_fn=create_line_criteria, ) - # Restore original agent output - agent_execution.agent_output = original_agent_output + workload_execution.workload_output = original_agent_output - # Build and return the aggregated result using utility function return build_line_by_line_result( line_details=line_details, line_results=line_results, diff --git a/packages/uipath/src/uipath/eval/evaluators/tool_call_args_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/tool_call_args_evaluator.py index 2703e3c76..bd10702ff 100644 --- a/packages/uipath/src/uipath/eval/evaluators/tool_call_args_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/tool_call_args_evaluator.py @@ -4,7 +4,12 @@ extract_tool_calls, tool_calls_args_score, ) -from ..models import AgentExecution, EvaluationResult, NumericEvaluationResult, ToolCall +from ..models import ( + EvaluationResult, + NumericEvaluationResult, + ToolCall, + WorkloadExecution, +) from ..models.models import EvaluatorType from .base_evaluator import ( BaseEvaluationCriteria, @@ -54,21 +59,21 @@ def get_evaluator_id(cls) -> str: async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: ToolCallArgsEvaluationCriteria, ) -> EvaluationResult: """Evaluate if the tool calls are in the correct order. Args: - agent_execution: The execution details containing: + workload_execution: The execution details containing: - agent_input: The input received by the agent - - agent_output: The final output of the agent - - agent_trace: The execution spans to use for the evaluation + - workload_output: The final output of the agent + - workload_trace: The execution spans to use for the evaluation evaluation_criteria: The criteria to evaluate Returns: EvaluationResult: Boolean result indicating correct tool call order (True/False) """ - tool_calls_order = extract_tool_calls(agent_execution.agent_trace) + tool_calls_order = extract_tool_calls(workload_execution.workload_trace) score, justification = tool_calls_args_score( tool_calls_order, evaluation_criteria.tool_calls, diff --git a/packages/uipath/src/uipath/eval/evaluators/tool_call_count_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/tool_call_count_evaluator.py index 11d684ae1..4657eec26 100644 --- a/packages/uipath/src/uipath/eval/evaluators/tool_call_count_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/tool_call_count_evaluator.py @@ -1,12 +1,11 @@ """Tool call count evaluator for validating expected tool usage patterns.""" -from collections import Counter - from .._helpers.evaluators_helpers import ( - extract_tool_calls_names, + count_tool_calls_by_name_and_id, + extract_tool_calls, tool_calls_count_score, ) -from ..models import AgentExecution, EvaluationResult, NumericEvaluationResult +from ..models import EvaluationResult, NumericEvaluationResult, WorkloadExecution from ..models.models import EvaluatorType from .base_evaluator import ( BaseEvaluationCriteria, @@ -58,22 +57,22 @@ def get_evaluator_id(cls) -> str: async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: ToolCallCountEvaluationCriteria, ) -> EvaluationResult: """Evaluate if the tool calls are in the correct order. Args: - agent_execution: The execution details containing: + workload_execution: The execution details containing: - agent_input: The input received by the agent - - agent_output: The final output of the agent - - agent_trace: The execution spans to use for the evaluation + - workload_output: The final output of the agent + - workload_trace: The execution spans to use for the evaluation evaluation_criteria: The criteria to evaluate Returns: EvaluationResult: Boolean result indicating correct tool call order (True/False) """ - tool_calls_count = Counter( - extract_tool_calls_names(agent_execution.agent_trace) + tool_calls_count = count_tool_calls_by_name_and_id( + extract_tool_calls(workload_execution.workload_trace, include_args=False) ) score, justification = tool_calls_count_score( tool_calls_count, diff --git a/packages/uipath/src/uipath/eval/evaluators/tool_call_order_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/tool_call_order_evaluator.py index 1050ddc76..17912081c 100644 --- a/packages/uipath/src/uipath/eval/evaluators/tool_call_order_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/tool_call_order_evaluator.py @@ -1,10 +1,10 @@ """Tool call order evaluator for validating correct sequence of tool calls.""" from .._helpers.evaluators_helpers import ( - extract_tool_calls_names, - tool_calls_order_score, + extract_tool_calls, + tool_calls_order_score_with_ids, ) -from ..models import AgentExecution, EvaluationResult, NumericEvaluationResult +from ..models import EvaluationResult, NumericEvaluationResult, WorkloadExecution from ..models.models import EvaluatorType from .base_evaluator import ( BaseEvaluationCriteria, @@ -55,23 +55,25 @@ def get_evaluator_id(cls) -> str: async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: ToolCallOrderEvaluationCriteria, ) -> EvaluationResult: """Evaluate if the tool calls are in the correct order. Args: - agent_execution: The execution details containing: + workload_execution: The execution details containing: - agent_input: The input received by the agent - - agent_output: The final output of the agent - - agent_trace: The execution spans to use for the evaluation + - workload_output: The final output of the agent + - workload_trace: The execution spans to use for the evaluation evaluation_criteria: The criteria to evaluate Returns: EvaluationResult: Boolean result indicating correct tool call order (True/False) """ - tool_calls_order = extract_tool_calls_names(agent_execution.agent_trace) - score, justification = tool_calls_order_score( - tool_calls_order, + actual_calls = extract_tool_calls( + workload_execution.workload_trace, include_args=False + ) + score, justification = tool_calls_order_score_with_ids( + actual_calls, evaluation_criteria.tool_calls_order, self.evaluator_config.strict, ) diff --git a/packages/uipath/src/uipath/eval/evaluators/tool_call_output_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/tool_call_output_evaluator.py index fff139daf..c098fac3c 100644 --- a/packages/uipath/src/uipath/eval/evaluators/tool_call_output_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/tool_call_output_evaluator.py @@ -5,10 +5,10 @@ tool_calls_output_score, ) from ..models import ( - AgentExecution, EvaluationResult, NumericEvaluationResult, ToolOutput, + WorkloadExecution, ) from ..models.models import EvaluatorType from .base_evaluator import ( @@ -60,21 +60,23 @@ def get_evaluator_id(cls) -> str: async def evaluate( self, - agent_execution: AgentExecution, + workload_execution: WorkloadExecution, evaluation_criteria: ToolCallOutputEvaluationCriteria, ) -> EvaluationResult: """Evaluate if the tool calls are in the correct order. Args: - agent_execution: The execution details containing: + workload_execution: The execution details containing: - agent_input: The input received by the agent - - agent_output: The final output of the agent - - agent_trace: The execution spans to use for the evaluation + - workload_output: The final output of the agent + - workload_trace: The execution spans to use for the evaluation evaluation_criteria: The criteria to evaluate Returns: EvaluationResult: Boolean result indicating correct tool call order (True/False) """ - tool_calls_outputs = extract_tool_calls_outputs(agent_execution.agent_trace) + tool_calls_outputs = extract_tool_calls_outputs( + workload_execution.workload_trace + ) score, justification = tool_calls_output_score( tool_calls_outputs, evaluation_criteria.tool_outputs, diff --git a/packages/uipath/src/uipath/eval/evaluators_types/ExactMatchEvaluator.json b/packages/uipath/src/uipath/eval/evaluators_types/ExactMatchEvaluator.json index 866b06416..5c302ccf8 100644 --- a/packages/uipath/src/uipath/eval/evaluators_types/ExactMatchEvaluator.json +++ b/packages/uipath/src/uipath/eval/evaluators_types/ExactMatchEvaluator.json @@ -2,6 +2,50 @@ "evaluatorTypeId": "uipath-exact-match", "evaluatorConfigSchema": { "$defs": { + "ConfusionMatrixAggregatorSpec": { + "description": "Run-level raw k\u00d7k confusion matrix \u2014 no scalar headline, no averaging.", + "properties": { + "type": { + "const": "confusion_matrix", + "default": "confusion_matrix", + "title": "Type", + "type": "string" + } + }, + "title": "ConfusionMatrixAggregatorSpec", + "type": "object" + }, + "FScoreAggregatorSpec": { + "description": "Run-level F-beta aggregator (multiclass, micro or macro averaged).", + "properties": { + "type": { + "const": "fscore", + "default": "fscore", + "title": "Type", + "type": "string" + }, + "averaging": { + "enum": [ + "macro", + "micro" + ], + "title": "Averaging", + "type": "string" + }, + "f_value": { + "default": 1.0, + "exclusiveMinimum": 0, + "maximum": 1000, + "title": "F Value", + "type": "number" + } + }, + "required": [ + "averaging" + ], + "title": "FScoreAggregatorSpec", + "type": "object" + }, "OutputEvaluationCriteria": { "description": "Base class for all output evaluation criteria.", "properties": { @@ -23,6 +67,54 @@ ], "title": "OutputEvaluationCriteria", "type": "object" + }, + "PrecisionAggregatorSpec": { + "description": "Run-level precision aggregator (multiclass, micro or macro averaged).", + "properties": { + "type": { + "const": "precision", + "default": "precision", + "title": "Type", + "type": "string" + }, + "averaging": { + "enum": [ + "macro", + "micro" + ], + "title": "Averaging", + "type": "string" + } + }, + "required": [ + "averaging" + ], + "title": "PrecisionAggregatorSpec", + "type": "object" + }, + "RecallAggregatorSpec": { + "description": "Run-level recall aggregator (multiclass, micro or macro averaged).", + "properties": { + "type": { + "const": "recall", + "default": "recall", + "title": "Type", + "type": "string" + }, + "averaging": { + "enum": [ + "macro", + "micro" + ], + "title": "Averaging", + "type": "string" + } + }, + "required": [ + "averaging" + ], + "title": "RecallAggregatorSpec", + "type": "object" } }, "description": "Configuration for the exact match evaluator.", @@ -50,9 +142,31 @@ "default": null }, "target_output_key": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], "default": "*", - "description": "Key to extract output from agent execution", - "title": "Target Output Key", + "description": "Key or list of keys to extract output from workload execution", + "title": "Target Output Key" + }, + "line_by_line_evaluator": { + "default": false, + "description": "If True, split output by delimiter and evaluate each line separately", + "title": "Line By Line Evaluator", + "type": "boolean" + }, + "line_delimiter": { + "default": "\n", + "description": "Delimiter to split output when line_by_line_evaluator is True", + "title": "Line Delimiter", "type": "string" }, "case_sensitive": { @@ -64,6 +178,60 @@ "default": false, "title": "Negated", "type": "boolean" + }, + "classes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Label vocabulary shared by every aggregator on this evaluator. Labels are matched case-insensitively against the per-datapoint expected/actual outputs.", + "title": "Classes" + }, + "aggregators": { + "anyOf": [ + { + "items": { + "discriminator": { + "mapping": { + "confusion_matrix": "#/$defs/ConfusionMatrixAggregatorSpec", + "fscore": "#/$defs/FScoreAggregatorSpec", + "precision": "#/$defs/PrecisionAggregatorSpec", + "recall": "#/$defs/RecallAggregatorSpec" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/$defs/PrecisionAggregatorSpec" + }, + { + "$ref": "#/$defs/RecallAggregatorSpec" + }, + { + "$ref": "#/$defs/FScoreAggregatorSpec" + }, + { + "$ref": "#/$defs/ConfusionMatrixAggregatorSpec" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Dataset-level metrics (precision / recall / F-score / confusion matrix) computed over the per-datapoint match outcomes. Requires ``classes``.", + "title": "Aggregators" } }, "title": "ExactMatchEvaluatorConfig", @@ -91,5 +259,23 @@ "title": "OutputEvaluationCriteria", "type": "object" }, - "justificationSchema": {} + "justificationSchema": { + "description": "Base class for all evaluator justifications.", + "properties": { + "expected": { + "title": "Expected", + "type": "string" + }, + "actual": { + "title": "Actual", + "type": "string" + } + }, + "required": [ + "expected", + "actual" + ], + "title": "BaseEvaluatorJustification", + "type": "object" + } } \ No newline at end of file diff --git a/packages/uipath/src/uipath/eval/helpers.py b/packages/uipath/src/uipath/eval/helpers.py index 0a0a0ca7f..167858813 100644 --- a/packages/uipath/src/uipath/eval/helpers.py +++ b/packages/uipath/src/uipath/eval/helpers.py @@ -7,6 +7,8 @@ from pydantic import ValidationError +from uipath.runtime.schema import UiPathRuntimeSchema + from .evaluators.base_evaluator import GenericBaseEvaluator from .evaluators.evaluator_factory import EvaluatorFactory from .mocks._types import InputMockingStrategy, LLMMockingStrategy @@ -210,6 +212,8 @@ def migrate_evaluation_item( for evaluation in eval_set.evaluations ], model_settings=eval_set.model_settings, + agent_memory_enabled=eval_set.agent_memory_enabled, + agent_memory_settings=eval_set.agent_memory_settings, ) except ValidationError as e: raise ValueError( @@ -277,3 +281,24 @@ async def load_evaluators( ) return evaluators + + +def get_agent_model(schema: UiPathRuntimeSchema) -> str | None: + """Get agent model from the runtime schema metadata. + + The model is read from schema.metadata["settings"]["model"] which is + populated by the low-code agents runtime from agent.json. + + Returns: + The model name from agent settings, or None if not found. + """ + try: + if schema.metadata and "settings" in schema.metadata: + settings = schema.metadata["settings"] + model = settings.get("model") + if model: + logger.debug(f"Got agent model from schema.metadata: {model}") + return model + return None + except Exception: + return None diff --git a/packages/uipath/src/uipath/eval/mocks/__init__.py b/packages/uipath/src/uipath/eval/mocks/__init__.py index 95dfb877e..ddb76ca70 100644 --- a/packages/uipath/src/uipath/eval/mocks/__init__.py +++ b/packages/uipath/src/uipath/eval/mocks/__init__.py @@ -1,14 +1,39 @@ """Mock interface.""" from ._mock_context import is_tool_simulated -from ._mock_runtime import UiPathMockRuntime -from ._types import ExampleCall, MockingContext +from ._mock_runtime import ( + UiPathMockRuntime, + build_mocking_context, + build_mocking_context_from_dict, +) +from ._types import ( + ComponentSimulationConfig, + ExampleCall, + MockingContext, + RuleOperator, + SimulationAnswer, + SimulationAnswerType, + SimulationBehavior, + SimulationCondition, + SimulationConfig, + SimulationStrategy, +) from .mockable import mockable __all__ = [ + "ComponentSimulationConfig", "ExampleCall", - "UiPathMockRuntime", "MockingContext", - "mockable", + "RuleOperator", + "SimulationAnswer", + "SimulationAnswerType", + "SimulationBehavior", + "SimulationCondition", + "SimulationConfig", + "SimulationStrategy", + "UiPathMockRuntime", + "build_mocking_context", + "build_mocking_context_from_dict", "is_tool_simulated", + "mockable", ] diff --git a/packages/uipath/src/uipath/eval/mocks/_input_mocker.py b/packages/uipath/src/uipath/eval/mocks/_input_mocker.py index f1d253ba8..a542fc7ad 100644 --- a/packages/uipath/src/uipath/eval/mocks/_input_mocker.py +++ b/packages/uipath/src/uipath/eval/mocks/_input_mocker.py @@ -1,6 +1,7 @@ """LLM Input Mocker implementation.""" import json +import logging from datetime import datetime from typing import Any @@ -9,14 +10,18 @@ from uipath.core.tracing import traced from uipath.platform import UiPath from uipath.platform.chat import UiPathLlmChatService +from uipath.platform.chat._llm_gateway_service import ChatModels from .._execution_context import eval_set_run_id_context from ._mock_context import cache_manager_context from ._mocker import UiPathInputMockingError +from ._structured_output import generate_structured_output from ._types import ( InputMockingStrategy, ) +logger = logging.getLogger(__name__) + def get_input_mocking_prompt( input_schema: str, @@ -101,15 +106,6 @@ async def generate_llm_input( prompt = get_input_mocking_prompt(**prompt_generation_args) - response_format = { - "type": "json_schema", - "json_schema": { - "name": "agent_input", - "strict": False, - "schema": input_schema, - }, - } - model_parameters = mocking_strategy.model if mocking_strategy else None completion_kwargs = ( model_parameters.model_dump(by_alias=False, exclude_none=True) @@ -117,9 +113,14 @@ async def generate_llm_input( else {} ) + simulation_model = completion_kwargs.get( + "model", ChatModels.gpt_4_1_mini_2025_04_14 + ) + logger.info(f"Simulating input generation using model: {simulation_model}") + if cache_manager is not None: cache_key_data = { - "response_format": response_format, + "input_schema": input_schema, "completion_kwargs": completion_kwargs, "prompt_generation_args": prompt_generation_args, } @@ -133,15 +134,15 @@ async def generate_llm_input( if cached_response is not None: return cached_response - response = await llm.chat_completions( + result = await generate_structured_output( + llm, [{"role": "user", "content": prompt}], - response_format=response_format, - **completion_kwargs, + schema=input_schema, + response_format_name="agent_input", + description="Return the simulated agent input matching the required schema.", + completion_kwargs=completion_kwargs, ) - generated_input_str = response.choices[0].message.content - result = json.loads(generated_input_str) - if cache_manager is not None: cache_manager.set( mocker_type="input_mocker", @@ -151,10 +152,6 @@ async def generate_llm_input( ) return result - except json.JSONDecodeError as e: - raise UiPathInputMockingError( - f"Failed to parse LLM response as JSON: {str(e)}" - ) from e except UiPathInputMockingError: raise except Exception as e: diff --git a/packages/uipath/src/uipath/eval/mocks/_llm_mocker.py b/packages/uipath/src/uipath/eval/mocks/_llm_mocker.py index 194aa6c09..5c9a0cf38 100644 --- a/packages/uipath/src/uipath/eval/mocks/_llm_mocker.py +++ b/packages/uipath/src/uipath/eval/mocks/_llm_mocker.py @@ -7,10 +7,11 @@ from opentelemetry.sdk.trace import ReadableSpan from pydantic import BaseModel, TypeAdapter +from uipath.core.serialization import serialize_defaults from uipath.core.tracing import traced from uipath.platform import UiPath from uipath.platform.chat import UiPathLlmChatService -from uipath.platform.chat._llm_gateway_service import _cleanup_schema +from uipath.platform.chat._llm_gateway_service import ChatModels, _cleanup_schema from .._execution_context import ( eval_set_run_id_context, @@ -28,6 +29,7 @@ UiPathMockResponseGenerationError, UiPathNoMockFoundError, ) +from ._structured_output import generate_structured_output from ._types import ( ExampleCall, LLMMockingStrategy, @@ -96,11 +98,16 @@ def __init__(self, context: MockingContext): @traced(name="__mocker__", recording=False) async def response( - self, func: Callable[[T], R], params: dict[str, Any], *args: T, **kwargs + self, + func: Callable[[T], R], + params: dict[str, Any], + invocation: tuple[tuple[Any, ...], dict[str, Any]], ) -> R: """Respond with mocked response generated by an LLM.""" assert isinstance(self.context.strategy, LLMMockingStrategy) + args, kwargs = invocation + function_name = params.get("name") or func.__name__ if function_name in [x.name for x in self.context.strategy.tools_to_simulate]: uipath = UiPath() @@ -120,14 +127,7 @@ async def response( "output_schema", TypeAdapter(return_type).json_schema() ) - response_format = { - "type": "json_schema", - "json_schema": { - "name": "OutputSchema", - "strict": False, - "schema": _cleanup_schema(output_schema), - }, - } + cleaned_schema = _cleanup_schema(output_schema) try: # Safely pull examples from params. example_calls = params.get("example_calls", []) @@ -172,7 +172,7 @@ async def response( "testRunProctorInstructions": self.context.strategy.prompt, } prompt_generation_args = { - k: json.dumps(pydantic_to_dict_safe(v)) + k: json.dumps(pydantic_to_dict_safe(v), default=serialize_defaults) for k, v in prompt_input.items() } model_parameters = self.context.strategy.model @@ -182,10 +182,17 @@ async def response( else {} ) + simulation_model = completion_kwargs.get( + "model", ChatModels.gpt_4_1_mini_2025_04_14 + ) + logger.info( + f"Simulating tool '{function_name}' using model: {simulation_model}" + ) + formatted_prompt = PROMPT.format(**prompt_generation_args) cache_key_data = { - "response_format": response_format, + "output_schema": cleaned_schema, "completion_kwargs": completion_kwargs, "prompt_generation_args": prompt_generation_args, } @@ -201,17 +208,17 @@ async def response( if cached_response is not None: return cached_response - response = await llm.chat_completions( - [ - { - "role": "user", - "content": formatted_prompt, - }, - ], - response_format=response_format, - **completion_kwargs, + result = await generate_structured_output( + llm, + [{"role": "user", "content": formatted_prompt}], + schema=cleaned_schema, + response_format_name="OutputSchema", + description=( + "Return the simulated response for tool " + f"'{function_name}' matching the required schema." + ), + completion_kwargs=completion_kwargs, ) - result = json.loads(response.choices[0].message.content) if cache_manager is not None: cache_manager.set( @@ -223,7 +230,7 @@ async def response( return result except Exception as e: - raise UiPathMockResponseGenerationError() from e + raise UiPathMockResponseGenerationError(str(e)) from e else: raise UiPathNoMockFoundError(f"Method '{function_name}' is not simulated.") diff --git a/packages/uipath/src/uipath/eval/mocks/_mock_context.py b/packages/uipath/src/uipath/eval/mocks/_mock_context.py index c2335544f..bd2f80df6 100644 --- a/packages/uipath/src/uipath/eval/mocks/_mock_context.py +++ b/packages/uipath/src/uipath/eval/mocks/_mock_context.py @@ -43,17 +43,25 @@ def is_tool_simulated(tool_name: str) -> bool: to be simulated, False otherwise. """ ctx = mocking_context.get() - strategy = ctx.strategy if ctx else None - if strategy is None: + if ctx is None: return False normalized_tool_name = _normalize_tool_name(tool_name) + if ctx.components: + return any( + _normalize_tool_name(c.component_id) == normalized_tool_name + for c in ctx.components + ) + + strategy = ctx.strategy + if strategy is None: + return False + if isinstance(strategy, LLMMockingStrategy): - simulated_names = [ + return normalized_tool_name in [ _normalize_tool_name(t.name) for t in strategy.tools_to_simulate ] - return normalized_tool_name in simulated_names elif isinstance(strategy, MockitoMockingStrategy): return any( _normalize_tool_name(b.function) == normalized_tool_name @@ -64,11 +72,13 @@ def is_tool_simulated(tool_name: str) -> bool: async def get_mocked_response( - func: Callable[[Any], Any], params: dict[str, Any], *args, **kwargs + func: Callable[[Any], Any], + params: dict[str, Any], + invocation: tuple[tuple[Any, ...], dict[str, Any]], ) -> Any: """Get a mocked response.""" mocker = mocker_context.get() if mocker is None: raise UiPathNoMockFoundError() else: - return await mocker.response(func, params, *args, **kwargs) + return await mocker.response(func, params, invocation) diff --git a/packages/uipath/src/uipath/eval/mocks/_mock_runtime.py b/packages/uipath/src/uipath/eval/mocks/_mock_runtime.py index 512d8d6ee..60cc78c2b 100644 --- a/packages/uipath/src/uipath/eval/mocks/_mock_runtime.py +++ b/packages/uipath/src/uipath/eval/mocks/_mock_runtime.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json import logging import uuid from collections.abc import AsyncGenerator @@ -28,13 +27,88 @@ LLMMockingStrategy, MockingContext, MockingStrategyType, - ToolSimulation, + ModelSettings, + SimulationConfig, ) logger = logging.getLogger(__name__) -def load_simulation_config() -> MockingContext | None: +def build_mocking_context( + config: SimulationConfig, agent_model: str | None = None +) -> MockingContext | None: + """Build a MockingContext from a validated SimulationConfig.""" + if not config.enabled: + return None + + # New per-component format → routes to simulate-component API + if config.components: + from uipath.platform.common._config import UiPathConfig + + workload_id = ( + getattr(UiPathConfig, "agent_id", None) + or getattr(UiPathConfig, "project_id", None) + or str(uuid.uuid4()) + ) + logger.debug( + f"Loaded simulation config for {len(config.components)} component(s)" + ) + return MockingContext( + strategy=None, + name="debug-simulation", + inputs={}, + components=config.components, + workload_id=workload_id, + ) + + # Legacy format (toolsToSimulate + instructions) → routes to local LLM mocker + if not config.tools_to_simulate: + return None + + model = ( + ModelSettings(model=config.model) + if config.model + else ModelSettings(model=agent_model) + if agent_model + else None + ) + + mocking_strategy = LLMMockingStrategy( + type=MockingStrategyType.LLM, + prompt=config.instructions, + tools_to_simulate=config.tools_to_simulate, + model=model, + ) + + logger.debug( + f"Loaded simulation config for {len(config.tools_to_simulate)} tool(s)" + ) + return MockingContext( + strategy=mocking_strategy, + name="debug-simulation", + inputs={}, + ) + + +def build_mocking_context_from_dict( + simulation_data: dict[str, Any], agent_model: str | None = None +) -> MockingContext | None: + """Build a MockingContext from a simulation config dictionary. + + Deprecated: prefer build_mocking_context with a validated SimulationConfig. + + Args: + simulation_data: Parsed simulation config (same schema as simulation.json). + agent_model: Optional agent model name to use as fallback. + + Returns: + MockingContext if valid and enabled, None otherwise. + """ + config = SimulationConfig.model_validate(simulation_data) + return build_mocking_context(config, agent_model) + + +def load_simulation_config(agent_model: str | None = None) -> MockingContext | None: """Load simulation.json from current directory and convert to MockingContext. Returns: @@ -47,38 +121,10 @@ def load_simulation_config() -> MockingContext | None: return None try: - with open(simulation_path, "r", encoding="utf-8") as f: - simulation_data = json.load(f) - - # Check if simulation is enabled - if not simulation_data.get("enabled", True): - return None - - # Extract tools to simulate - tools_to_simulate = [ - ToolSimulation(name=tool["name"]) - for tool in simulation_data.get("toolsToSimulate", []) - ] - - if not tools_to_simulate: - return None - - # Create LLM mocking strategy - mocking_strategy = LLMMockingStrategy( - type=MockingStrategyType.LLM, - prompt=simulation_data.get("instructions", ""), - tools_to_simulate=tools_to_simulate, + config = SimulationConfig.model_validate_json( + simulation_path.read_text(encoding="utf-8") ) - - # Create MockingContext for debugging - mocking_context = MockingContext( - strategy=mocking_strategy, - name="debug-simulation", - inputs={}, - ) - - logger.info(f"Loaded simulation config for {len(tools_to_simulate)} tool(s)") - return mocking_context + return build_mocking_context(config, agent_model) except Exception as e: logger.warning(f"Failed to load simulation.json: {e}") @@ -95,12 +141,16 @@ def set_execution_context( mocking_context.set(context) try: - if context and context.strategy: - mocker_context.set(MockerFactory.create(context)) + if context and (context.strategy or context.components): + mocker = MockerFactory.create(context) + mocker_context.set(mocker) + logger.info( + "simulate-component: mocker created (%s)", type(mocker).__name__ + ) else: mocker_context.set(None) except Exception: - logger.warning("Failed to create mocker.") + logger.warning("Failed to create mocker.", exc_info=True) mocker_context.set(None) span_collector_context.set(span_collector) diff --git a/packages/uipath/src/uipath/eval/mocks/_mocker.py b/packages/uipath/src/uipath/eval/mocks/_mocker.py index 57cb8bcc3..99e5da1b2 100644 --- a/packages/uipath/src/uipath/eval/mocks/_mocker.py +++ b/packages/uipath/src/uipath/eval/mocks/_mocker.py @@ -16,8 +16,7 @@ async def response( self, func: Callable[[T], R], params: dict[str, Any], - *args: T, - **kwargs, + invocation: tuple[tuple[Any, ...], dict[str, Any]], ) -> R: """Respond with mocked response.""" raise NotImplementedError() diff --git a/packages/uipath/src/uipath/eval/mocks/_mocker_factory.py b/packages/uipath/src/uipath/eval/mocks/_mocker_factory.py index 4c001bfe0..2f61162a4 100644 --- a/packages/uipath/src/uipath/eval/mocks/_mocker_factory.py +++ b/packages/uipath/src/uipath/eval/mocks/_mocker_factory.py @@ -3,6 +3,7 @@ from ._llm_mocker import LLMMocker from ._mocker import Mocker from ._mockito_mocker import MockitoMocker +from ._simulate_component_mocker import SimulateComponentMocker from ._types import ( LLMMockingStrategy, MockingContext, @@ -16,6 +17,8 @@ class MockerFactory: @staticmethod def create(context: MockingContext) -> Mocker: """Create a mocker instance.""" + if context.components: + return SimulateComponentMocker(context) match context.strategy: case LLMMockingStrategy(): return LLMMocker(context) diff --git a/packages/uipath/src/uipath/eval/mocks/_mockito_mocker.py b/packages/uipath/src/uipath/eval/mocks/_mockito_mocker.py index 041478baf..a9b30230f 100644 --- a/packages/uipath/src/uipath/eval/mocks/_mockito_mocker.py +++ b/packages/uipath/src/uipath/eval/mocks/_mockito_mocker.py @@ -99,12 +99,17 @@ def __init__(self, context: MockingContext): stubbed = stubbed.thenRaise(_resolve_value(answer_dict["value"])) async def response( - self, func: Callable[[T], R], params: dict[str, Any], *args: T, **kwargs + self, + func: Callable[[T], R], + params: dict[str, Any], + invocation: tuple[tuple[Any, ...], dict[str, Any]], ) -> R: """Return mocked response or raise appropriate errors.""" if not isinstance(self.context.strategy, MockitoMockingStrategy): raise UiPathMockResponseGenerationError("Mocking strategy misconfigured.") + args, kwargs = invocation + # No behavior configured → call real function is_mocked = any( behavior.function == params["name"] diff --git a/packages/uipath/src/uipath/eval/mocks/_simulate_component_mocker.py b/packages/uipath/src/uipath/eval/mocks/_simulate_component_mocker.py new file mode 100644 index 000000000..f35359ec4 --- /dev/null +++ b/packages/uipath/src/uipath/eval/mocks/_simulate_component_mocker.py @@ -0,0 +1,143 @@ +"""Mocker that routes tool calls through the simulate-component API.""" + +from __future__ import annotations + +import logging +from typing import Any, Callable, cast + +from pydantic import TypeAdapter + +from uipath.platform.chat._llm_gateway_service import _cleanup_schema + +from .._execution_context import execution_id_context, span_collector_context +from ._llm_mocker import LLMMocker +from ._mocker import ( + Mocker, + R, + T, + UiPathMockResponseGenerationError, + UiPathNoMockFoundError, +) +from ._simulate_component_service import _create_simulate_component_service +from ._types import ComponentSimulationConfig, MockingContext + +logger = logging.getLogger(__name__) + + +class SimulateComponentMocker(Mocker): + """Routes each tool call to the simulate-component API based on per-component config.""" + + def __init__(self, context: MockingContext) -> None: + self._context = context + self._components: dict[str, ComponentSimulationConfig] = { + c.component_id: c for c in (context.components or []) + } + self._normalized: dict[str, ComponentSimulationConfig] = { + c.component_id.replace("_", " "): c for c in (context.components or []) + } + self._workload_id = context.workload_id or "" + + def _find_component(self, tool_name: str) -> ComponentSimulationConfig | None: + return self._components.get(tool_name) or self._normalized.get( + tool_name.replace("_", " ") + ) + + async def response( + self, + func: Callable[[T], R], + params: dict[str, Any], + invocation: tuple[tuple[Any, ...], dict[str, Any]], + ) -> R: + tool_name = params.get("name") or func.__name__ + component = self._find_component(tool_name) + + if component is None: + raise UiPathNoMockFoundError(f"No simulation config for '{tool_name}'.") + + args, kwargs = invocation + + return_type: Any = func.__annotations__.get("return", None) or Any + raw_output_schema = ( + params.get("output_schema") or TypeAdapter(return_type).json_schema() + ) + output_schema = component.output_schema or _cleanup_schema(raw_output_schema) + input_payload = {"args": list(args), "kwargs": kwargs} + input_schema = component.input_schema or params.get("input_schema") + + execution_history = self._build_execution_history() + trace_id, parent_span_id = self._get_span_context() + workload_info = { + "name": self._context.name, + "userInput": self._context.inputs, + } + + example_calls = [ + {"id": ex.id, "input": ex.input, "output": ex.output} + for ex in (params.get("example_calls") or []) + ] + + payload: dict[str, Any] = { + "workloadId": self._workload_id, + "componentId": component.component_id, + "componentType": component.component_type or "tool", + "componentDescription": component.component_description + or params.get("description"), + "input": input_payload, + "inputSchema": input_schema, + "outputSchema": output_schema, + "simulationInstruction": component.simulation_instruction, + "simulationStrategy": int(component.simulation_strategy), + "mockValue": component.mock_value, + "behaviors": ( + [b.model_dump() for b in component.behaviors] + if component.behaviors + else None + ), + "exampleCalls": example_calls or None, + "executionHistory": execution_history or None, + "workloadInfo": workload_info, + "traceId": trace_id, + "parentSpanId": parent_span_id, + } + + logger.info("simulate-component: calling API for '%s'", tool_name) + try: + service = _create_simulate_component_service() + result = await service.simulate(payload) + except Exception as e: + logger.error( + "simulate-component: API call failed for '%s': %s", tool_name, e + ) + raise UiPathMockResponseGenerationError( + f"simulate-component API call failed for '{tool_name}'" + ) from e + + status = result.get("status") + if status == 1: # Completed + logger.info("simulate-component: '%s' simulated successfully", tool_name) + return cast(R, result.get("simulatedOutput")) + + error = result.get("error") or {} + error_message = error.get("message", f"Simulation failed for '{tool_name}'") + logger.error("simulate-component: '%s' failed — %s", tool_name, error_message) + raise UiPathMockResponseGenerationError(error_message) + + def _build_execution_history(self) -> str | None: + span_collector = span_collector_context.get() + execution_id = execution_id_context.get() + if span_collector and execution_id: + spans = span_collector.get_spans(execution_id) + return LLMMocker.spans_to_llm_context(spans) if spans else None + return None + + @staticmethod + def _get_span_context() -> tuple[str | None, str | None]: + """Return (traceId, parentSpanId) from the current OTel span, or (None, None).""" + from opentelemetry import trace + + span_ctx = trace.get_current_span().get_span_context() + if not span_ctx.is_valid: + return None, None + trace_id = f"{span_ctx.trace_id:032x}" + span_id = f"{span_ctx.span_id:016x}" + return trace_id, span_id diff --git a/packages/uipath/src/uipath/eval/mocks/_simulate_component_service.py b/packages/uipath/src/uipath/eval/mocks/_simulate_component_service.py new file mode 100644 index 000000000..50f83cad1 --- /dev/null +++ b/packages/uipath/src/uipath/eval/mocks/_simulate_component_service.py @@ -0,0 +1,43 @@ +"""Service for calling the simulate-component API.""" + +import json +from typing import Any + +from uipath._utils import Endpoint +from uipath.core.serialization import serialize_json +from uipath.platform.common import BaseService +from uipath.platform.constants import ( + HEADER_INTERNAL_ACCOUNT_ID, + HEADER_INTERNAL_TENANT_ID, +) + + +class SimulateComponentService(BaseService): + async def simulate(self, payload: dict[str, Any]) -> dict[str, Any]: + from uipath.platform.common import UiPathConfig + + headers: dict[str, str] = {} + if UiPathConfig.tenant_id: + headers[HEADER_INTERNAL_TENANT_ID] = UiPathConfig.tenant_id + if UiPathConfig.organization_id: + headers[HEADER_INTERNAL_ACCOUNT_ID] = UiPathConfig.organization_id + + response = await self.request_async( + "POST", + url=Endpoint( + "/agentsruntime_/api/execution/simulations/simulate-component" + ), + json=json.loads(serialize_json(payload)), + headers=headers, + ) + return response.json() + + +def _create_simulate_component_service() -> SimulateComponentService: + from uipath.platform import UiPath + + uipath = UiPath() + return SimulateComponentService( + config=uipath._config, + execution_context=uipath._execution_context, + ) diff --git a/packages/uipath/src/uipath/eval/mocks/_structured_output.py b/packages/uipath/src/uipath/eval/mocks/_structured_output.py new file mode 100644 index 000000000..599780353 --- /dev/null +++ b/packages/uipath/src/uipath/eval/mocks/_structured_output.py @@ -0,0 +1,259 @@ +"""Provider-aware structured output for the eval mockers. + +The normalized LLM Gateway handles OpenAI-style ``response_format`` +(json_schema) differently per provider — live-verified against the gateway: + +- **OpenAI**: honors ``response_format`` and returns valid JSON content, + including native ``$defs`` support. +- **Anthropic (Claude)**: ignores it and answers with plain prose content. +- **Gemini**: returns empty content. + +Forced function calling works across all three providers, so each provider +gets a small strategy class: OpenAI prefers ``response_format`` (more reliable +for it on some schemas) with a tool-call fallback; Claude and Gemini go +straight to the forced tool call; unknown providers try ``response_format`` +first and fall back. +""" + +import json +import logging +from typing import Any + +from uipath.platform.chat.llm_gateway import RequiredToolChoice + +RESPONSE_TOOL_NAME = "submit_tool_response" +RESPONSE_KEY = "response" +_DEFS_PREFIX = "#/$defs/" + +logger = logging.getLogger(__name__) + + +def _inline_defs( + schema: dict[str, Any], +) -> tuple[dict[str, Any], dict[str, Any]]: + """Inline ``$defs``/``$ref`` into a self-contained schema. + + Nested Pydantic models and enums emit root ``$defs`` referenced by ``$ref``. + The normalized gateway accepts those in ``response_format`` but not inside a + tool's ``parameters``, so they are inlined here. Sibling keys on a ``$ref`` + node (e.g. a field ``description``) are merged over the inlined definition. + Self-referential definitions cannot be inlined without looping; any ``$ref`` + reached while its target is already on the current resolution path is left + untouched and its definitions are returned so the caller can keep them + reachable. + + Returns: + A tuple of (inlined schema, leftover ``$defs`` needed for cyclic refs). + """ + defs = schema.get("$defs", {}) + leftover: dict[str, Any] = {} + + def resolve(node: Any, active: frozenset[str]) -> Any: + if isinstance(node, dict): + ref = node.get("$ref") + if isinstance(ref, str) and ref.startswith(_DEFS_PREFIX): + name = ref[len(_DEFS_PREFIX) :] + if name in defs and name not in active: + resolved = resolve(defs[name], active | {name}) + siblings = { + key: resolve(value, active) + for key, value in node.items() + if key not in ("$ref", "$defs") + } + if isinstance(resolved, dict): + return {**resolved, **siblings} + return resolved + # Cyclic or unknown ref: keep it and preserve its definition. + if name in defs: + leftover[name] = defs[name] + return dict(node) + return { + key: resolve(value, active) + for key, value in node.items() + if key != "$defs" + } + if isinstance(node, list): + return [resolve(item, active) for item in node] + return node + + root = {key: value for key, value in schema.items() if key != "$defs"} + inlined = resolve(root, frozenset()) + return inlined, leftover + + +def build_response_tool(schema: dict[str, Any], description: str) -> dict[str, Any]: + """Build a normalized-API function tool that wraps ``schema`` under ``response``. + + Tool-call arguments are always a JSON object, so an arbitrary output schema + (which may be a scalar, array, or object) is nested under a single + ``response`` property and unwrapped after the call. ``$defs``/``$ref`` are + inlined so the tool parameters are self-contained, which the gateway requires + for tool schemas (unlike ``response_format``). + """ + response_schema, leftover_defs = _inline_defs(schema) + parameters: dict[str, Any] = { + "type": "object", + "properties": {RESPONSE_KEY: response_schema}, + "required": [RESPONSE_KEY], + } + if leftover_defs: + parameters["$defs"] = leftover_defs + + return { + "name": RESPONSE_TOOL_NAME, + "description": description, + "parameters": parameters, + } + + +def extract_response(response: Any) -> Any: + """Extract the wrapped value from the forced tool call. + + Raises: + ValueError: if the response carries no usable tool call or is missing the + wrapped ``response`` key. + """ + choices = getattr(response, "choices", None) + if not choices: + raise ValueError("LLM response contained no choices") + + message = choices[0].message + tool_calls = getattr(message, "tool_calls", None) + if not tool_calls: + raise ValueError( + f"LLM response contained no tool calls (content={message.content!r})" + ) + + arguments = tool_calls[0].arguments + if RESPONSE_KEY not in arguments: + raise ValueError( + f"Tool call arguments missing '{RESPONSE_KEY}' key: {arguments}" + ) + + return arguments[RESPONSE_KEY] + + +class ToolCallStructuredOutput: + """Structured output via a forced tool call — works on every provider.""" + + async def generate( + self, + llm: Any, + messages: list[dict[str, str]], + *, + schema: dict[str, Any], + response_format_name: str, + description: str, + completion_kwargs: dict[str, Any], + ) -> Any: + """Force a tool call wrapping ``schema`` and unwrap its arguments.""" + tool = build_response_tool(schema, description) + response = await llm.chat_completions( + messages, + tools=[tool], + tool_choice=RequiredToolChoice(), + **completion_kwargs, + ) + return extract_response(response) + + +class ResponseFormatStructuredOutput(ToolCallStructuredOutput): + """Prefer ``response_format`` (json_schema); fall back to a forced tool call. + + The fallback fires when the provider rejects the request, returns empty + content, or returns content that is not valid JSON (Claude's behavior on + the normalized gateway is to answer with plain prose). + """ + + async def generate( + self, + llm: Any, + messages: list[dict[str, str]], + *, + schema: dict[str, Any], + response_format_name: str, + description: str, + completion_kwargs: dict[str, Any], + ) -> Any: + """Try ``response_format`` first, falling back to a forced tool call.""" + response_format = { + "type": "json_schema", + "json_schema": { + "name": response_format_name, + "strict": False, + "schema": schema, + }, + } + + content: str | None = None + try: + response = await llm.chat_completions( + messages, response_format=response_format, **completion_kwargs + ) + choices = getattr(response, "choices", None) + if choices: + content = choices[0].message.content + except Exception as e: + logger.info("response_format path failed, falling back to tools: %s", e) + + if content: + try: + return json.loads(content) + except json.JSONDecodeError: + logger.info( + "response_format content was not JSON, falling back to tools" + ) + + return await super().generate( + llm, + messages, + schema=schema, + response_format_name=response_format_name, + description=description, + completion_kwargs=completion_kwargs, + ) + + +class OpenAIStructuredOutput(ResponseFormatStructuredOutput): + """OpenAI honors ``response_format`` natively (including ``$defs``).""" + + +class AnthropicStructuredOutput(ToolCallStructuredOutput): + """Claude answers ``response_format`` with prose; go straight to tools.""" + + +class GeminiStructuredOutput(ToolCallStructuredOutput): + """Gemini returns empty content for ``response_format``; go straight to tools.""" + + +def _strategy_for_model(model: str | None) -> ToolCallStructuredOutput: + name = (model or "").lower() + if "claude" in name or name.startswith("anthropic"): + return AnthropicStructuredOutput() + if "gemini" in name: + return GeminiStructuredOutput() + if name.startswith(("gpt", "o1", "o3", "o4")): + return OpenAIStructuredOutput() + # Unknown providers: try response_format, fall back to tools. + return ResponseFormatStructuredOutput() + + +async def generate_structured_output( + llm: Any, + messages: list[dict[str, str]], + *, + schema: dict[str, Any], + response_format_name: str, + description: str, + completion_kwargs: dict[str, Any], +) -> Any: + """Generate structured output using the strategy for the requested model.""" + strategy = _strategy_for_model(completion_kwargs.get("model")) + return await strategy.generate( + llm, + messages, + schema=schema, + response_format_name=response_format_name, + description=description, + completion_kwargs=completion_kwargs, + ) diff --git a/packages/uipath/src/uipath/eval/mocks/_types.py b/packages/uipath/src/uipath/eval/mocks/_types.py index 827569879..f1a15312c 100644 --- a/packages/uipath/src/uipath/eval/mocks/_types.py +++ b/packages/uipath/src/uipath/eval/mocks/_types.py @@ -121,12 +121,142 @@ class UnknownMockingStrategy(BaseMockingStrategy): MockingStrategy = Union[KnownMockingStrategy, UnknownMockingStrategy] +# --------------------------------------------------------------------------- +# Per-component simulation types — mirror the simulate-component API contract +# --------------------------------------------------------------------------- + + +class SimulationStrategy(int, Enum): + """Simulation strategy matching the simulate-component API. + + Integer values are part of the cross-language API contract — do not reorder. + """ + + LLM = 0 + MOCKITO = 1 + STATIC = 2 + + +class RuleOperator(int, Enum): + """Comparison operator for Mockito condition matching. + + Integer values are part of the cross-language API contract — do not reorder. + """ + + EQ = 0 + NE = 1 + GT = 2 + GTE = 3 + LT = 4 + LTE = 5 + CONTAINS = 6 + + +class SimulationAnswerType(int, Enum): + """Answer type for a Mockito simulation behavior. + + Integer values are part of the cross-language API contract — do not reorder. + """ + + RETURN = 0 + RAISE = 1 + + +class SimulationAnswer(BaseModel): + type: SimulationAnswerType = SimulationAnswerType.RETURN + value: Any = None + + model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + + +class SimulationCondition(BaseModel): + field: str + op: RuleOperator = RuleOperator.EQ + value: Any = None + + model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + + +class SimulationBehavior(BaseModel): + when: list[SimulationCondition] | None = None + then: list[SimulationAnswer] + + model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + + +class ComponentSimulationConfig(BaseModel): + """Per-component simulation config matching the simulate-component API request schema. + + Runtime-injected fields (workloadId, runId, input, traceId, parentSpanId, + folderKey) are supplied at call time. inputSchema and outputSchema can be + overridden here; if omitted they are derived from the function annotations. + """ + + component_id: str = Field(..., alias="componentId") + component_type: str | None = Field(None, alias="componentType") + component_description: str | None = Field(None, alias="componentDescription") + simulation_instruction: str | None = Field(None, alias="simulationInstruction") + simulation_strategy: SimulationStrategy = Field( + SimulationStrategy.LLM, alias="simulationStrategy" + ) + mock_value: Any = Field(None, alias="mockValue") + behaviors: list[SimulationBehavior] | None = None + input_schema: dict[str, Any] | None = Field(None, alias="inputSchema") + output_schema: dict[str, Any] | None = Field(None, alias="outputSchema") + + model_config = ConfigDict( + validate_by_name=True, validate_by_alias=True, extra="allow" + ) + + class MockingContext(BaseModel): """Execution context for mocking, holding strategy and inputs.""" strategy: MockingStrategy | None inputs: dict[str, Any] = Field(default_factory=lambda: {}) name: str = Field(default="debug") + # When set, SimulateComponentMocker routes each tool call to the simulate-component API. + components: list[ComponentSimulationConfig] | None = None + workload_id: str | None = None + + +class SimulationConfig(BaseModel): + """Top-level schema for simulation.json / --simulation flag. + + New format (routes to simulate-component API): + { + "enabled": true, + "components": [ + { + "componentId": "my_tool", + "componentType": "tool", + "simulationStrategy": 0, + "simulationInstruction": "Simulate this tool by..." + } + ] + } + + Legacy format (routes to local LLM mocker): + { + "enabled": true, + "toolsToSimulate": [{"name": "my_tool"}], + "instructions": "Simulate these tools by..." + } + """ + + enabled: bool = True + # New per-component format — when non-empty, routes to simulate-component API. + components: list[ComponentSimulationConfig] = Field(default_factory=list) + # Legacy flat format — used when components is empty; routes to local LLM mocker. + tools_to_simulate: list[ToolSimulation] = Field( + default_factory=list, alias="toolsToSimulate" + ) + instructions: str = "" + model: str | None = None + + model_config = ConfigDict( + validate_by_name=True, validate_by_alias=True, extra="allow" + ) class ExampleCall(BaseModel): diff --git a/packages/uipath/src/uipath/eval/mocks/mockable.py b/packages/uipath/src/uipath/eval/mocks/mockable.py index 254f88b89..3e9a324b9 100644 --- a/packages/uipath/src/uipath/eval/mocks/mockable.py +++ b/packages/uipath/src/uipath/eval/mocks/mockable.py @@ -39,7 +39,7 @@ def mocked_response_decorator(func, params: dict[str, Any]): """Mocked response decorator.""" async def mock_response_generator(*args, **kwargs): - mocked_response = await get_mocked_response(func, params, *args, **kwargs) + mocked_response = await get_mocked_response(func, params, (args, kwargs)) # Mocking successful. context = UiPathSpanUtils.get_parent_context() diff --git a/packages/uipath/src/uipath/eval/models/__init__.py b/packages/uipath/src/uipath/eval/models/__init__.py index 580ce812a..efc0d1f95 100644 --- a/packages/uipath/src/uipath/eval/models/__init__.py +++ b/packages/uipath/src/uipath/eval/models/__init__.py @@ -1,7 +1,9 @@ """UiPath evaluation module for agent performance assessment.""" +import warnings +from typing import Any + from uipath.eval.models.models import ( - AgentExecution, BooleanEvaluationResult, ErrorEvaluationResult, EvalItemResult, @@ -15,10 +17,11 @@ ScoreType, ToolCall, ToolOutput, + WorkloadExecution, ) __all__ = [ - "AgentExecution", + "WorkloadExecution", "EvaluationResult", "EvaluationResultDto", "LLMResponse", @@ -31,6 +34,23 @@ "NumericEvaluationResult", "ErrorEvaluationResult", "ToolCall", - "EvaluatorType", "ToolOutput", ] + +# Backward-compatibility shim: ``AgentExecution`` was renamed to +# ``WorkloadExecution``. The old name keeps working but emits a +# DeprecationWarning. Remove in uipath 3.0. +_DEPRECATED_NAMES = {"AgentExecution": "WorkloadExecution"} + + +def __getattr__(name: str) -> Any: + new_name = _DEPRECATED_NAMES.get(name) + if new_name is not None: + warnings.warn( + f"{name} is deprecated and will be removed in uipath 3.0; " + f"use {new_name} instead.", + DeprecationWarning, + stacklevel=2, + ) + return globals()[new_name] + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/packages/uipath/src/uipath/eval/models/_conversational_utils.py b/packages/uipath/src/uipath/eval/models/_conversational_utils.py index d4dbf0cdd..9e3523acc 100644 --- a/packages/uipath/src/uipath/eval/models/_conversational_utils.py +++ b/packages/uipath/src/uipath/eval/models/_conversational_utils.py @@ -168,7 +168,6 @@ def legacy_conversational_eval_input_to_uipath_message_list( role="user", content_parts=content_parts, tool_calls=[], - interrupts=[], created_at=timestamp, updated_at=timestamp, ) @@ -215,7 +214,6 @@ def legacy_conversational_eval_input_to_uipath_message_list( role="assistant", content_parts=content_parts, tool_calls=tool_calls, - interrupts=[], created_at=timestamp, updated_at=timestamp, ) @@ -259,7 +257,6 @@ def legacy_conversational_eval_input_to_uipath_message_list( role="user", content_parts=content_parts, tool_calls=[], - interrupts=[], created_at=timestamp, updated_at=timestamp, ) @@ -301,7 +298,6 @@ def legacy_conversational_eval_output_to_uipath_message_data_list( role="assistant", content_parts=content_parts, tool_calls=tool_calls, - interrupts=[], ) ) diff --git a/packages/uipath/src/uipath/eval/models/evaluation_set.py b/packages/uipath/src/uipath/eval/models/evaluation_set.py index 22e6ce244..889f0f0e9 100644 --- a/packages/uipath/src/uipath/eval/models/evaluation_set.py +++ b/packages/uipath/src/uipath/eval/models/evaluation_set.py @@ -1,8 +1,9 @@ """Evaluation set models.""" +import re from typing import Any, Literal -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, field_validator from pydantic.alias_generators import to_camel from ..mocks._types import ( @@ -15,6 +16,21 @@ LegacyConversationalEvalOutput, ) +_GUID_RE = re.compile( + r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" + r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" +) + + +def normalize_eval_id(value: str) -> str: + """Canonicalize a GUID id to lowercase; leave non-GUID ids unchanged. + + GUIDs are case-insensitive, but downstream correlation (selection, + span/cache keying) compares ids as plain strings, so a mixed-case id + must be normalized at ingestion to stay matchable. + """ + return value.lower() if isinstance(value, str) and _GUID_RE.match(value) else value + class EvaluatorReference(BaseModel): """Reference to an evaluator with optional weight. @@ -73,6 +89,22 @@ class EvaluationSetModelSettings(BaseModel): temperature: float | str | None = Field(default=None, alias="temperature") +class EvaluationSetAgentMemorySettings(BaseModel): + """Agent memory setting overrides within evaluation sets with ID. + + Values are stored as strings; the literal "same-as-agent" preserves the + agent's own memory configuration for that field. Matches the Agents + eval-set storage schema (agentMemorySettings). + """ + + model_config = ConfigDict(populate_by_name=True) + + id: str = Field(..., alias="id") + result_count: str = Field(default="same-as-agent", alias="resultCount") + search_mode: str = Field(default="same-as-agent", alias="searchMode") + threshold: str = Field(default="same-as-agent", alias="threshold") + + class EvaluationItem(BaseModel): """Individual evaluation item within an evaluation set.""" @@ -96,6 +128,12 @@ class EvaluationItem(BaseModel): alias="inputMockingStrategy", ) + @field_validator("id") + @classmethod + def _normalize_id(cls, value: str) -> str: + """Normalize GUID ids to canonical lowercase.""" + return normalize_eval_id(value) + class LegacyEvaluationItem(BaseModel): """Individual evaluation item within an evaluation set.""" @@ -130,6 +168,12 @@ class LegacyEvaluationItem(BaseModel): default=None, alias="conversationalExpectedOutput" ) + @field_validator("id") + @classmethod + def _normalize_id(cls, value: str) -> str: + """Normalize GUID ids to canonical lowercase.""" + return normalize_eval_id(value) + class EvaluationSet(BaseModel): """Complete evaluation set model.""" @@ -149,11 +193,15 @@ class EvaluationSet(BaseModel): model_settings: list[EvaluationSetModelSettings] = Field( default_factory=list, alias="modelSettings" ) + agent_memory_enabled: bool = Field(default=False, alias="agentMemoryEnabled") + agent_memory_settings: list[EvaluationSetAgentMemorySettings] = Field( + default_factory=list, alias="agentMemorySettings" + ) def extract_selected_evals(self, eval_ids) -> None: """Filter evaluations to only include those with specified IDs.""" selected_evals: list[EvaluationItem] = [] - remaining_ids = set(eval_ids) + remaining_ids = {normalize_eval_id(eval_id) for eval_id in eval_ids} for evaluation in self.evaluations: if evaluation.id in remaining_ids: selected_evals.append(evaluation) @@ -181,13 +229,17 @@ class LegacyEvaluationSet(BaseModel): model_settings: list[EvaluationSetModelSettings] = Field( default_factory=list, alias="modelSettings" ) + agent_memory_enabled: bool = Field(default=False, alias="agentMemoryEnabled") + agent_memory_settings: list[EvaluationSetAgentMemorySettings] = Field( + default_factory=list, alias="agentMemorySettings" + ) created_at: str = Field(alias="createdAt") updated_at: str = Field(alias="updatedAt") def extract_selected_evals(self, eval_ids) -> None: """Filter evaluations to only include those with specified IDs.""" selected_evals: list[LegacyEvaluationItem] = [] - remaining_ids = set(eval_ids) + remaining_ids = {normalize_eval_id(eval_id) for eval_id in eval_ids} for evaluation in self.evaluations: if evaluation.id in remaining_ids: selected_evals.append(evaluation) diff --git a/packages/uipath/src/uipath/eval/models/models.py b/packages/uipath/src/uipath/eval/models/models.py index d2dc26df9..b3c2f1700 100644 --- a/packages/uipath/src/uipath/eval/models/models.py +++ b/packages/uipath/src/uipath/eval/models/models.py @@ -1,6 +1,7 @@ """Models for evaluation framework including execution data and evaluation results.""" import traceback +import warnings from dataclasses import dataclass from enum import Enum, IntEnum from typing import Annotated, Any, Literal, Union @@ -11,14 +12,14 @@ from pydantic_core import core_schema -class AgentExecution(BaseModel): - """Represents the execution data of an agent for evaluation purposes.""" +class WorkloadExecution(BaseModel): + """Represents the execution data of a workload for evaluation purposes.""" model_config = ConfigDict(arbitrary_types_allowed=True) agent_input: dict[str, Any] | None - agent_output: dict[str, Any] | str - agent_trace: list[ReadableSpan] + workload_output: dict[str, Any] | str + workload_trace: list[ReadableSpan] expected_agent_behavior: str | None = None simulation_instructions: str = "" @@ -175,7 +176,7 @@ def from_int(cls, value: int) -> "LegacyEvaluatorType": class TrajectoryEvaluationSpan: """Simplified span representation for trajectory evaluation. - Contains span information needed for evaluating agent execution paths, + Contains span information needed for evaluating workload execution paths, excluding timestamps which are not useful for trajectory analysis. """ @@ -303,17 +304,29 @@ class EvaluatorType(str, Enum): class ToolCall(BaseModel): - """Represents a tool call with its arguments.""" + """Represents a tool call with its arguments. + + `id` is the stable identifier from the tool's resource definition (e.g. a + UUID from `bindings.json`). When present on both the actual call and the + expected criterion, scorers match by `id` so a rename of `name` does not + break eval sets. When `id` is absent on either side, scorers fall back to + matching by `name` (the legacy behavior). + """ name: str args: dict[str, Any] + id: str | None = None class ToolOutput(BaseModel): - """Represents a tool output with its output.""" + """Represents a tool output with its output. + + See `ToolCall.id` for the id semantics. + """ name: str output: str + id: str | None = None class UiPathEvaluationErrorCategory(str, Enum): @@ -376,3 +389,23 @@ def __init__( def as_dict(self) -> dict[str, Any]: """Get the error information as a dictionary.""" return self.error_info.model_dump() + + +# ── Backward-compatibility shim ────────────────────────────────────────────── +# ``AgentExecution`` was renamed to ``WorkloadExecution`` (unified-evals naming). +# Per the release policy's deprecation requirements, the old name keeps working +# but emits a DeprecationWarning when accessed. Remove this shim in uipath 3.0. +_DEPRECATED_NAMES = {"AgentExecution": "WorkloadExecution"} + + +def __getattr__(name: str) -> Any: + new_name = _DEPRECATED_NAMES.get(name) + if new_name is not None: + warnings.warn( + f"{name} is deprecated and will be removed in uipath 3.0; " + f"use {new_name} instead.", + DeprecationWarning, + stacklevel=2, + ) + return globals()[new_name] + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/packages/uipath/src/uipath/eval/runtime/_spans.py b/packages/uipath/src/uipath/eval/runtime/_spans.py index 6f492388e..eec742f26 100644 --- a/packages/uipath/src/uipath/eval/runtime/_spans.py +++ b/packages/uipath/src/uipath/eval/runtime/_spans.py @@ -414,7 +414,7 @@ async def configure_evaluation_span( evaluation_run_results: UiPathEvalRunResult object containing evaluation results execution_id: The execution ID for this evaluation input_data: The input data for this evaluation - agent_execution_output: Optional agent execution output for error checking + agent_execution_output: Optional workload execution output for error checking """ # Extract evaluator scores (already normalized to 0-100) evaluator_scores = extract_evaluator_scores(evaluation_run_results) diff --git a/packages/uipath/src/uipath/eval/runtime/_types.py b/packages/uipath/src/uipath/eval/runtime/_types.py index 2aee5e599..fa84f0d9e 100644 --- a/packages/uipath/src/uipath/eval/runtime/_types.py +++ b/packages/uipath/src/uipath/eval/runtime/_types.py @@ -1,7 +1,7 @@ import logging from opentelemetry.sdk.trace import ReadableSpan -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, Field from pydantic.alias_generators import to_camel from uipath.runtime import UiPathRuntimeResult @@ -78,6 +78,9 @@ class UiPathEvalOutput(BaseModel): evaluation_set_name: str evaluation_set_results: list[UiPathEvalRunResult] + dataset_evaluator_results: dict[str, EvaluationResultDto] = Field( + default_factory=dict + ) @property def score(self) -> float: diff --git a/packages/uipath/src/uipath/eval/runtime/events.py b/packages/uipath/src/uipath/eval/runtime/events.py index 589f82ba7..2a46469e4 100644 --- a/packages/uipath/src/uipath/eval/runtime/events.py +++ b/packages/uipath/src/uipath/eval/runtime/events.py @@ -29,6 +29,11 @@ class EvalSetRunCreatedEvent(BaseModel): eval_set_id: str eval_set_run_id: str | None = None no_of_evals: int + # Coarse agent-type label the runtime factory declared via + # ``UiPathRuntimeFactorySettings.agent_type``. Consumers stamp this + # onto telemetry verbatim -- the CLI does not classify entrypoints. + # ``None`` when the factory has no opinion. + agent_type: str | None = None # skip validation to avoid abstract class instantiation evaluators: SkipValidation[list[GenericBaseEvaluator[Any, Any, Any]]] diff --git a/packages/uipath/src/uipath/eval/runtime/runtime.py b/packages/uipath/src/uipath/eval/runtime/runtime.py index 1c32b9efe..2f089cf7b 100644 --- a/packages/uipath/src/uipath/eval/runtime/runtime.py +++ b/packages/uipath/src/uipath/eval/runtime/runtime.py @@ -38,6 +38,7 @@ UiPathRuntimeStorageProtocol, ) from uipath.runtime.errors import ( + UiPathBaseRuntimeError, UiPathErrorCategory, UiPathErrorContract, ) @@ -45,21 +46,31 @@ from uipath.runtime.schema import UiPathRuntimeSchema from .._execution_context import ExecutionSpanCollector -from ..evaluators.base_evaluator import GenericBaseEvaluator +from ..evaluators.base_evaluator import ( + BaseEvaluatorJustification, + GenericBaseEvaluator, +) +from ..evaluators.dataset_evaluator_factory import ( + build_dataset_evaluator, + dataset_result_key, + unique_aggregator_specs, +) +from ..evaluators.exact_match_evaluator import ExactMatchEvaluatorConfig from ..evaluators.output_evaluator import OutputEvaluationCriteria +from ..helpers import get_agent_model from ..mocks._cache_manager import CacheManager from ..mocks._input_mocker import ( generate_llm_input, ) from ..mocks._mock_context import cache_manager_context from ..mocks._mock_runtime import UiPathMockRuntime -from ..mocks._types import MockingContext +from ..mocks._types import LLMMockingStrategy, MockingContext, ModelSettings from ..models import EvaluationResult from ..models.evaluation_set import ( EvaluationItem, EvaluationSet, ) -from ..models.models import AgentExecution, EvalItemResult, EvaluationResultDto +from ..models.models import EvalItemResult, EvaluationResultDto, WorkloadExecution from ._exporters import ( ExecutionLogsExporter, ExecutionSpanExporter, @@ -95,6 +106,19 @@ logger = logging.getLogger(__name__) +def _is_user_facing_error(exception: BaseException) -> bool: + """Whether an exception is a correctly-reported workload failure. + + User-category errors (e.g. invalid input, business logic failures) are + failures of the agent under evaluation, not eval infrastructure crashes, + so they must not be flagged as runtime exceptions. + """ + return ( + isinstance(exception, UiPathBaseRuntimeError) + and exception.error_info.category == UiPathErrorCategory.USER + ) + + def compute_evaluator_scores( evaluation_set_results: list[UiPathEvalRunResult], evaluators: Iterable[GenericBaseEvaluator[Any, Any, Any]], @@ -201,6 +225,95 @@ def compute_evaluator_scores( return final_score, agg_metrics_per_evaluator +def compute_dataset_evaluator_results( + evaluation_set_results: list[UiPathEvalRunResult], + evaluators: Iterable[GenericBaseEvaluator[Any, Any, Any]], +) -> dict[str, EvaluationResultDto]: + """Run any dataset-level aggregators embedded in per-datapoint evaluator configs. + + Walks ``evaluators`` looking for any whose config carries an ``aggregators`` + list (currently only ExactMatch). For each aggregator spec, builds the + corresponding dataset evaluator via the factory and runs it over the + per-datapoint results that came from that source evaluator. + + Args: + evaluation_set_results: Per-datapoint results from the run. + evaluators: Per-datapoint evaluator instances that ran during this eval + set. Their configs may carry ``aggregators`` lists. + + Returns: + Dict keyed by :func:`dataset_result_key` (same scheme as the platform + worker), with each value's ``details`` dumped to the camelCase wire + shape. Exact-duplicate specs are deduped; aggregators whose source + produced no results still emit a zeroed result. + """ + # Deduplicate by (datapoint, evaluator) before aggregating, mirroring + # compute_evaluator_scores. The partial-failure path (see _execute_eval's + # except block) can emit a zero-score, details-less DTO for an evaluator + # that already produced a real result on the success path; and an external + # retry/resume can re-feed a datapoint. Without dedup those inflate + # n_total/n_skipped and would double-count real matrix pairs. When a + # datapoint has multiple DTOs for one evaluator, prefer the one whose + # details parse into an expected/actual justification. + latest_by_dp_eval: dict[tuple[str, str], EvaluationResultDto] = {} + for eval_run_result in evaluation_set_results: + datapoint_id = eval_run_result.evaluation_name + for eval_run_result_dto in eval_run_result.evaluation_run_results: + if eval_run_result_dto.is_line_result: + continue + dedup_key = (datapoint_id, eval_run_result_dto.evaluator_name) + existing = latest_by_dp_eval.get(dedup_key) + candidate = eval_run_result_dto.result + # Keep the entry with a parseable justification over one without. + if existing is not None and ( + BaseEvaluatorJustification.try_from(candidate.details) is None + or BaseEvaluatorJustification.try_from(existing.details) is not None + ): + continue + latest_by_dp_eval[dedup_key] = candidate + + results_by_evaluator: defaultdict[str, list[EvaluationResultDto]] = defaultdict( + list + ) + for (_dp_id, dedup_eval_name), dp_result in latest_by_dp_eval.items(): + results_by_evaluator[dedup_eval_name].append(dp_result) + + dataset_results: dict[str, EvaluationResultDto] = {} + for evaluator in evaluators: + # Aggregators currently only live on ExactMatch evaluator configs — the + # per-datapoint match outcome (with expected/actual labels in the + # justification) is exactly what the confusion matrix needs. Widen the + # isinstance tuple if a future evaluator type grows an ``aggregators`` + # field. + config = getattr(evaluator, "evaluator_config", None) + if not isinstance(config, ExactMatchEvaluatorConfig): + continue + if not config.aggregators or not config.classes: + continue + source_name = config.name + source_results = results_by_evaluator.get(source_name, []) + specs = unique_aggregator_specs(config.aggregators) + type_counts: dict[str, int] = defaultdict(int) + for spec in specs: + type_counts[spec.type] += 1 + for spec in specs: + dataset_evaluator = build_dataset_evaluator( + spec, source_name, config.classes + ) + key = dataset_result_key(source_name, spec, type_counts[spec.type] > 1) + result = dataset_evaluator.evaluate(source_results) + details: str | dict[str, Any] | None + if isinstance(result.details, BaseModel): + # Same camelCase wire shape the platform worker ships. + details = result.details.model_dump(by_alias=True, exclude_none=True) + else: + details = result.details + dataset_results[key] = EvaluationResultDto( + score=result.score, details=details + ) + return dataset_results + + class UiPathEvalRuntime: """Specialized runtime for evaluation runs, with access to the factory.""" @@ -285,6 +398,9 @@ async def initiate_evaluation( f"Please run with a single evaluation using --eval-ids to specify one evaluation." ) + factory_settings = await self.factory.get_settings() + agent_type = factory_settings.agent_type if factory_settings else None + await self.event_bus.publish( EvaluationEvents.CREATE_EVAL_SET_RUN, EvalSetRunCreatedEvent( @@ -293,6 +409,7 @@ async def initiate_evaluation( eval_set_run_id=self.context.eval_set_run_id, eval_set_id=self.context.evaluation_set.id, no_of_evals=len(self.context.evaluation_set.evaluations), + agent_type=agent_type, evaluators=self.context.evaluators, ), ) @@ -380,6 +497,14 @@ async def execute(self) -> UiPathRuntimeResult: evaluators, ) + # Run dataset-level aggregators over the per-datapoint results. + results.dataset_evaluator_results = ( + compute_dataset_evaluator_results( + results.evaluation_set_results, + evaluators, + ) + ) + # Configure span with output and metadata await configure_eval_set_run_span( span=span, @@ -526,12 +651,25 @@ async def _execute_eval( eval_item=eval_item, ), ) + # Set agent model on the mocking strategy if not already set + mocking_strategy = eval_item.mocking_strategy + if ( + mocking_strategy + and isinstance(mocking_strategy, LLMMockingStrategy) + and not mocking_strategy.model + ): + mocking_model = get_agent_model(self.context.runtime_schema) + if mocking_model: + mocking_strategy = mocking_strategy.model_copy( + update={"model": ModelSettings(model=mocking_model)} + ) + agent_execution_output = await self.execute_runtime( eval_item, execution_id, input_overrides=self.context.input_overrides, mocking_context=MockingContext( - strategy=eval_item.mocking_strategy, + strategy=mocking_strategy, name=eval_item.name, inputs=eval_item.inputs, ), @@ -539,10 +677,10 @@ async def _execute_eval( ) logger.debug( - f"DEBUG: Agent execution result status: {agent_execution_output.result.status}" + f"DEBUG: Workload execution result status: {agent_execution_output.result.status}" ) logger.debug( - f"DEBUG: Agent execution result trigger: {agent_execution_output.result.trigger}" + f"DEBUG: Workload execution result trigger: {agent_execution_output.result.trigger}" ) except Exception as e: @@ -750,7 +888,13 @@ async def _execute_eval( ) except Exception as e: - exception_details = EvalItemExceptionDetails(exception=e) + root_exception: Exception = ( + e.root_exception if isinstance(e, EvaluationRuntimeException) else e + ) + exception_details = EvalItemExceptionDetails( + exception=root_exception, + runtime_exception=not _is_user_facing_error(root_exception), + ) for evaluator in evaluators: evaluation_run_results.evaluation_run_results.append( @@ -775,13 +919,6 @@ async def _execute_eval( if isinstance(e, EvaluationRuntimeException): eval_run_updated_event.spans = e.spans eval_run_updated_event.logs = e.logs - if eval_run_updated_event.exception_details: - eval_run_updated_event.exception_details.exception = ( - e.root_exception - ) - eval_run_updated_event.exception_details.runtime_exception = ( - True - ) await self.event_bus.publish( EvaluationEvents.UPDATE_EVAL_RUN, @@ -811,8 +948,18 @@ async def _generate_input_for_eval( or getattr(eval_item, "expected_output", None) or {} ) + # Set agent model on the input mocking strategy if not already set + input_strategy = eval_item.input_mocking_strategy + # If input strategy does not specify a model, extract it + if input_strategy and not input_strategy.model: + input_generation_model = get_agent_model(self.context.runtime_schema) + if input_generation_model: + input_strategy = input_strategy.model_copy( + update={"model": ModelSettings(model=input_generation_model)} + ) + generated_input = await generate_llm_input( - eval_item.input_mocking_strategy, + input_strategy, (await self.get_schema()).input, expected_behavior=eval_item.expected_agent_behavior or "", expected_output=expected_output, @@ -993,16 +1140,20 @@ async def run_evaluator( else: output_data = execution_output.result.output - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input=eval_item.inputs, - agent_output=output_data, - agent_trace=execution_output.spans, + workload_output=output_data, + workload_trace=execution_output.spans, expected_agent_behavior=eval_item.expected_agent_behavior, ) + # Pass positionally so custom evaluators that still declare the old + # `agent_execution` parameter name keep working (the public keyword + # rename to `workload_execution` is a documented break — see the + # 2.12.0 migration notes). result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, - evaluation_criteria=evaluation_criteria, + workload_execution, + evaluation_criteria, ) # Create "Evaluation output" child span with the result diff --git a/packages/uipath/src/uipath/functions/__init__.py b/packages/uipath/src/uipath/functions/__init__.py index 74ffac411..43991807f 100644 --- a/packages/uipath/src/uipath/functions/__init__.py +++ b/packages/uipath/src/uipath/functions/__init__.py @@ -1,5 +1,6 @@ """UiPath Functions Runtime - factory and runtime for function-based execution.""" +from uipath.platform.constants import UIPATH_CONFIG_FILE from uipath.runtime import UiPathRuntimeFactoryRegistry from .debug import UiPathDebugFunctionsRuntime @@ -12,9 +13,9 @@ def register_default_runtime_factory(): UiPathRuntimeFactoryRegistry.register( "uipath", factory_callable=lambda context: UiPathFunctionsRuntimeFactory( - config_path="uipath.json", + config_path=UIPATH_CONFIG_FILE, ), - config_file="uipath.json", + config_file=UIPATH_CONFIG_FILE, ) UiPathRuntimeFactoryRegistry.set_default("uipath") diff --git a/packages/uipath/src/uipath/functions/factory.py b/packages/uipath/src/uipath/functions/factory.py index 2f9888469..3ef7f5b37 100644 --- a/packages/uipath/src/uipath/functions/factory.py +++ b/packages/uipath/src/uipath/functions/factory.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Any +from uipath.platform.constants import UIPATH_CONFIG_FILE from uipath.runtime import ( UiPathRuntimeFactorySettings, UiPathRuntimeProtocol, @@ -16,11 +17,21 @@ logger = logging.getLogger(__name__) +# Wire labels this factory advertises via +# :class:`UiPathRuntimeFactorySettings`. The runtime does not enumerate +# valid values -- each factory owns its own vocabulary and hosts +# forward them verbatim to telemetry / audit consumers. +_AGENT_TYPE_CODED = "uipath_coded" +# Functions runtime is plain Python — no third-party agent framework. +_AGENT_FRAMEWORK = "python" + class UiPathFunctionsRuntimeFactory: """Factory for discovering and creating function-based runtimes.""" - def __init__(self, config_path: str = "uipath.json", base_dir: str | None = None): + def __init__( + self, config_path: str = UIPATH_CONFIG_FILE, base_dir: str | None = None + ): """Initialize the factory with the path to uipath.json configuration.""" self.config_path = Path(config_path) self.base_dir = Path(base_dir) if base_dir else self.config_path.parent @@ -56,8 +67,16 @@ async def get_storage(self) -> UiPathRuntimeStorageProtocol | None: return None async def get_settings(self) -> UiPathRuntimeFactorySettings | None: - """Get factory settings for coded functions.""" - return None + """Get factory settings for coded functions. + + Advertises this factory's ``agent_type`` and ``agent_framework`` + wire labels so hosts (governance audit, App Insights telemetry) + can stamp them onto events without any host-side classification. + """ + return UiPathRuntimeFactorySettings( + agent_type=_AGENT_TYPE_CODED, + agent_framework=_AGENT_FRAMEWORK, + ) async def new_runtime( self, entrypoint: str, runtime_id: str, **kwargs diff --git a/packages/uipath/src/uipath/functions/runtime.py b/packages/uipath/src/uipath/functions/runtime.py index 3467d72b5..2e4b1045b 100644 --- a/packages/uipath/src/uipath/functions/runtime.py +++ b/packages/uipath/src/uipath/functions/runtime.py @@ -10,6 +10,8 @@ from types import ModuleType from typing import Any, AsyncGenerator, Callable, Type, cast, get_type_hints +from pydantic import ValidationError + from uipath.runtime import ( UiPathExecuteOptions, UiPathRuntimeEvent, @@ -35,6 +37,24 @@ logger = logging.getLogger(__name__) +def _format_input_validation_error( + function_name: str, input_type: Type[Any], error: Exception +) -> str: + """Build a concise, human-readable message for an input conversion failure.""" + type_name = getattr(input_type, "__name__", str(input_type)) + if isinstance(error, ValidationError): + issues = "; ".join( + f"{'.'.join(str(loc) for loc in err['loc']) or type_name}: {err['msg']}" + for err in error.errors() + ) + else: + issues = str(error) + return ( + f"Input does not match the expected schema for " + f"'{function_name}' ({type_name}): {issues}" + ) + + class UiPathFunctionsRuntime: """Runtime wrapper for a single Python function with full script executor compatibility.""" @@ -141,7 +161,18 @@ async def _execute_function( or is_pydantic_model(input_type) or (inspect.isclass(input_type) and hasattr(input_type, "__annotations__")) ): - typed_input = convert_to_class(input_data, cast(Type[Any], input_type)) + try: + typed_input = convert_to_class(input_data, cast(Type[Any], input_type)) + except (ValidationError, TypeError, ValueError) as e: + raise UiPathRuntimeError( + # Closest available code; uipath-runtime 0.12.x has no + # dedicated input-validation error code yet. + UiPathErrorCode.INPUT_INVALID_JSON, + "Invalid input", + _format_input_validation_error(self.function_name, input_type, e), + UiPathErrorCategory.USER, + include_traceback=False, + ) from e result = await func(typed_input) if is_async else func(typed_input) else: # Dict/untyped parameter diff --git a/packages/uipath/src/uipath/telemetry/__init__.py b/packages/uipath/src/uipath/telemetry/__init__.py index 92b3a354c..77b79bf1d 100644 --- a/packages/uipath/src/uipath/telemetry/__init__.py +++ b/packages/uipath/src/uipath/telemetry/__init__.py @@ -1,4 +1,7 @@ -from ._track import ( # noqa: D104 +"""UiPath telemetry tracking.""" + +from ._constants import PERIODIC_TELEMETRY_FLUSH_FEATURE_FLAG +from ._track import ( flush_events, is_telemetry_enabled, reset_event_client, @@ -8,6 +11,7 @@ ) __all__ = [ + "PERIODIC_TELEMETRY_FLUSH_FEATURE_FLAG", "track", "track_event", "is_telemetry_enabled", diff --git a/packages/uipath/src/uipath/telemetry/_constants.py b/packages/uipath/src/uipath/telemetry/_constants.py index 2583c6715..7a9104deb 100644 --- a/packages/uipath/src/uipath/telemetry/_constants.py +++ b/packages/uipath/src/uipath/telemetry/_constants.py @@ -1,5 +1,7 @@ _CONNECTION_STRING = "$CONNECTION_STRING" +PERIODIC_TELEMETRY_FLUSH_FEATURE_FLAG = "EnablePeriodicTelemetryFlush" + _APP_INSIGHTS_EVENT_MARKER_ATTRIBUTE = "APPLICATION_INSIGHTS_EVENT_MARKER_ATTRIBUTE" _OTEL_RESOURCE_ATTRIBUTES = "OTEL_RESOURCE_ATTRIBUTES" _SDK_VERSION = "SdkVersion" diff --git a/packages/uipath/src/uipath/telemetry/_track.py b/packages/uipath/src/uipath/telemetry/_track.py index 2d3f11ebf..8d09b69fc 100644 --- a/packages/uipath/src/uipath/telemetry/_track.py +++ b/packages/uipath/src/uipath/telemetry/_track.py @@ -1,20 +1,23 @@ import atexit import json import os +import threading from functools import wraps from importlib.metadata import version from logging import INFO, WARNING, LogRecord, getLogger -from typing import Any, Callable, ClassVar, Dict, Mapping, Optional, Union +from typing import Any, Callable, ClassVar, Dict, Mapping from opentelemetry.sdk._logs import LoggingHandler from opentelemetry.util.types import AnyValue -from .._utils.constants import ( +from uipath.core.feature_flags import FeatureFlags +from uipath.platform.constants import ( ENV_BASE_URL, ENV_ORGANIZATION_ID, ENV_TELEMETRY_ENABLED, ENV_TENANT_ID, ) + from ._constants import ( _APP_INSIGHTS_EVENT_MARKER_ATTRIBUTE, _APP_NAME, @@ -31,6 +34,7 @@ _SDK_VERSION, _TELEMETRY_CONFIG_FILE, _UNKNOWN, + PERIODIC_TELEMETRY_FLUSH_FEATURE_FLAG, ) # Try to import Application Insights client for custom events @@ -58,7 +62,7 @@ def _parse_connection_string( connection_string: str, -) -> Optional[Dict[str, str]]: +) -> Dict[str, str] | None: """Parse Azure Application Insights connection string. Args: @@ -89,6 +93,8 @@ def _parse_connection_string( _logger = getLogger(__name__) _logger.propagate = False +_PERIODIC_TELEMETRY_FLUSH_INTERVAL_SECONDS = 5.0 + def _get_connection_string() -> str | None: """Get the Application Insights connection string. @@ -105,18 +111,23 @@ def _get_connection_string() -> str | None: def _get_project_key() -> str: - """Get project key from telemetry file if present. + """Get the id used to attribute telemetry. - Returns: - Project key string if available, otherwise empty string. + Resolves ``uipath.json#id`` (then the runtime env var) via the shared + ``resolve_project_id`` helper, falling back to a legacy ``.uipath/.telemetry.json`` + ``ProjectKey`` if present. + Returns ``_UNKNOWN`` when no id is available. """ + from uipath.platform.common._span_utils import resolve_project_id + + if project_id := resolve_project_id(): + return project_id + try: telemetry_file = os.path.join(".uipath", _TELEMETRY_CONFIG_FILE) if os.path.exists(telemetry_file): with open(telemetry_file, "r") as f: - telemetry_data = json.load(f) - project_id = telemetry_data.get(_PROJECT_KEY) - if project_id: + if project_id := json.load(f).get(_PROJECT_KEY): return project_id except (json.JSONDecodeError, IOError, KeyError): pass @@ -224,13 +235,19 @@ class _AppInsightsEventClient: """ _initialized = False - _client: Optional[Any] = None + _client: Any | None = None _atexit_registered = False - _connection_string_provider: ClassVar[Optional[Callable[[], Optional[str]]]] = None + _connection_string_provider: ClassVar[Callable[[], str | None] | None] = None + _lifecycle_lock = threading.RLock() + _flush_lock = threading.Lock() + _flush_thread: threading.Thread | None = None + _flush_stop_event: threading.Event | None = None + _pending_events = threading.Event() + _shutdown_requested = False @staticmethod def set_connection_string_provider( - provider: Callable[[], Optional[str]], + provider: Callable[[], str | None], ) -> None: """Override how the connection string is resolved. @@ -242,59 +259,101 @@ def set_connection_string_provider( @staticmethod def _initialize() -> None: """Initialize Application Insights client for custom events.""" - if _AppInsightsEventClient._initialized: - return - - _AppInsightsEventClient._initialized = True + with _AppInsightsEventClient._lifecycle_lock: + if _AppInsightsEventClient._shutdown_requested: + return + if _AppInsightsEventClient._initialized: + return - # Suppress verbose logging from Application Insights SDK - # The SDK logs telemetry ingestion details which should not be user-facing - getLogger("applicationinsights").setLevel(WARNING) - getLogger("applicationinsights.channel").setLevel(WARNING) + _AppInsightsEventClient._initialized = True - if not _HAS_APPINSIGHTS: - return + # Suppress verbose logging from Application Insights SDK + # The SDK logs telemetry ingestion details which should not be user-facing + getLogger("applicationinsights").setLevel(WARNING) + getLogger("applicationinsights.channel").setLevel(WARNING) - if _AppInsightsEventClient._connection_string_provider: - connection_string = _AppInsightsEventClient._connection_string_provider() - else: - connection_string = _get_connection_string() - if not connection_string: - return + if not _HAS_APPINSIGHTS: + return - try: - parsed = _parse_connection_string(connection_string) - if not parsed: + if _AppInsightsEventClient._connection_string_provider: + connection_string = ( + _AppInsightsEventClient._connection_string_provider() + ) + else: + connection_string = _get_connection_string() + if not connection_string: return - instrumentation_key = parsed["InstrumentationKey"] - ingestion_endpoint = parsed.get("IngestionEndpoint") + try: + parsed = _parse_connection_string(connection_string) + if not parsed: + return + + instrumentation_key = parsed["InstrumentationKey"] + ingestion_endpoint = parsed.get("IngestionEndpoint") - # Build custom channel: DiagnosticSender → SynchronousQueue → TelemetryChannel - if ingestion_endpoint: - endpoint_url = ingestion_endpoint.rstrip("/") + "/v2/track" - else: - endpoint_url = None # SDK default + # Build custom channel: DiagnosticSender → SynchronousQueue → TelemetryChannel + if ingestion_endpoint: + endpoint_url = ingestion_endpoint.rstrip("/") + "/v2/track" + else: + endpoint_url = None # SDK default + + sender = _DiagnosticSender(service_endpoint_uri=endpoint_url) + queue = SynchronousQueue(sender) + channel = TelemetryChannel(queue=queue) + + _AppInsightsEventClient._client = AppInsightsTelemetryClient( + instrumentation_key, telemetry_channel=channel + ) - sender = _DiagnosticSender(service_endpoint_uri=endpoint_url) - queue = SynchronousQueue(sender) - channel = TelemetryChannel(queue=queue) + # Set application version + _AppInsightsEventClient._client.context.application.ver = version( + "uipath" + ) + except Exception as e: + # Log but don't raise - telemetry should never break the main application + _logger.warning( + f"Failed to initialize Application Insights client: {e}" + ) + _logger.debug( + "Application Insights initialization error", exc_info=True + ) - _AppInsightsEventClient._client = AppInsightsTelemetryClient( - instrumentation_key, telemetry_channel=channel + @staticmethod + def _ensure_periodic_flush_worker() -> None: + """Start the feature-gated periodic flush worker once.""" + if not FeatureFlags.is_flag_enabled(PERIODIC_TELEMETRY_FLUSH_FEATURE_FLAG): + return + + with _AppInsightsEventClient._lifecycle_lock: + if _AppInsightsEventClient._shutdown_requested: + return + current = _AppInsightsEventClient._flush_thread + if current and current.is_alive(): + return + + stop_event = threading.Event() + worker = threading.Thread( + target=_AppInsightsEventClient._periodic_flush_worker, + args=(stop_event,), + name="uipath-appinsights-flush", + daemon=True, ) + _AppInsightsEventClient._flush_stop_event = stop_event + _AppInsightsEventClient._flush_thread = worker + worker.start() - # Set application version - _AppInsightsEventClient._client.context.application.ver = version("uipath") - except Exception as e: - # Log but don't raise - telemetry should never break the main application - _logger.warning(f"Failed to initialize Application Insights client: {e}") - _logger.debug("Application Insights initialization error", exc_info=True) + @staticmethod + def _periodic_flush_worker(stop_event: threading.Event) -> None: + """Flush periodically until shutdown is requested.""" + while not stop_event.wait(_PERIODIC_TELEMETRY_FLUSH_INTERVAL_SECONDS): + if _AppInsightsEventClient._pending_events.is_set(): + _AppInsightsEventClient.flush() @staticmethod def track_event( name: str, - properties: Optional[Dict[str, Any]] = None, + properties: Dict[str, Any] | None = None, ) -> None: """Track a custom event to Application Insights customEvents table. @@ -304,62 +363,100 @@ def track_event( """ _AppInsightsEventClient._initialize() - if not _AppInsightsEventClient._client: - return + with _AppInsightsEventClient._lifecycle_lock: + if _AppInsightsEventClient._shutdown_requested: + return - try: - safe_properties: Dict[str, str] = {} - if properties: - for key, value in properties.items(): - if value is not None: - safe_properties[key] = str(value) - - _AppInsightsEventClient._client.track_event( - name=name, properties=safe_properties, measurements={} - ) - # Note: We don't flush after every event to avoid blocking. - # Events will be sent in batches by the SDK. - except Exception as e: - # Log but don't raise - telemetry should never break the main application - _logger.warning(f"Failed to track event '{name}': {e}") - _logger.debug(f"Event tracking error for '{name}'", exc_info=True) + if not _AppInsightsEventClient._client: + return + + _AppInsightsEventClient._ensure_periodic_flush_worker() + + try: + safe_properties: Dict[str, str] = {} + if properties: + for key, value in properties.items(): + if value is not None: + safe_properties[key] = str(value) + + client = _AppInsightsEventClient._client + if not client: + return + client.track_event( + name=name, properties=safe_properties, measurements={} + ) + _AppInsightsEventClient._pending_events.set() + # Note: We don't flush after every event to avoid blocking. + # Events are sent when the queue fills, on explicit/shutdown flush, + # or periodically when the periodic flush feature is enabled. + except Exception as e: + # Log but don't raise - telemetry should never break the main application + _logger.warning(f"Failed to track event '{name}': {e}") + _logger.debug(f"Event tracking error for '{name}'", exc_info=True) @staticmethod def flush() -> None: """Flush any pending telemetry events.""" - if _AppInsightsEventClient._client: - try: - _AppInsightsEventClient._client.flush() - # Check if items remain after flush (indicates send failure) + with _AppInsightsEventClient._flush_lock: + client = _AppInsightsEventClient._client + if client: + _AppInsightsEventClient._pending_events.clear() try: - remaining = ( - _AppInsightsEventClient._client.channel.queue._queue.qsize() - ) - if remaining > 0: - _logger.warning( - "AppInsights flush: %d items still in queue after flush", - remaining, - ) - except Exception: - pass - except Exception as e: - # Log but don't raise - telemetry should never break the main application - _logger.warning(f"Failed to flush telemetry events: {e}") - _logger.debug("Telemetry flush error", exc_info=True) + client.flush() + # Check if items remain after flush (indicates send failure) + try: + remaining = client.channel.queue._queue.qsize() + if remaining > 0: + _AppInsightsEventClient._pending_events.set() + _logger.warning( + "AppInsights flush: %d items still in queue after flush", + remaining, + ) + except Exception: + pass + except Exception as e: + _AppInsightsEventClient._pending_events.set() + # Log but don't raise - telemetry should never break the main application + _logger.warning(f"Failed to flush telemetry events: {e}") + _logger.debug("Telemetry flush error", exc_info=True) + + @staticmethod + def _shutdown(*, reset_client: bool = False) -> None: + """Stop the periodic worker and perform a synchronized final flush.""" + with _AppInsightsEventClient._lifecycle_lock: + _AppInsightsEventClient._shutdown_requested = True + stop_event = _AppInsightsEventClient._flush_stop_event + worker = _AppInsightsEventClient._flush_thread + if stop_event: + stop_event.set() + + if worker and worker is not threading.current_thread(): + worker.join() + + _AppInsightsEventClient.flush() + + if _AppInsightsEventClient._flush_thread is worker: + _AppInsightsEventClient._flush_thread = None + _AppInsightsEventClient._flush_stop_event = None + + if reset_client: + _AppInsightsEventClient._client = None + _AppInsightsEventClient._initialized = False + _AppInsightsEventClient._pending_events.clear() + _AppInsightsEventClient._shutdown_requested = False @staticmethod def register_atexit_flush() -> None: - """Register an atexit handler to flush events on process exit.""" - if not _AppInsightsEventClient._atexit_registered: - atexit.register(_AppInsightsEventClient.flush) - _AppInsightsEventClient._atexit_registered = True + """Register an atexit handler to stop the worker and flush events.""" + with _AppInsightsEventClient._lifecycle_lock: + if not _AppInsightsEventClient._atexit_registered: + atexit.register(_AppInsightsEventClient._shutdown) + _AppInsightsEventClient._atexit_registered = True @staticmethod def reset() -> None: - """Flush pending events and reset so the next call re-initializes.""" - _AppInsightsEventClient.flush() - _AppInsightsEventClient._client = None - _AppInsightsEventClient._initialized = False + """Flush pending events, stop the worker, and reset client state.""" + _AppInsightsEventClient._shutdown(reset_client=True) class _TelemetryClient: @@ -399,7 +496,7 @@ def _initialize(): _logger.debug("Telemetry initialization error", exc_info=True) @staticmethod - def _track_method(name: str, attrs: Optional[Dict[str, Any]] = None): + def _track_method(name: str, attrs: Dict[str, Any] | None = None): """Track function invocations using OpenTelemetry.""" if not _TelemetryClient._is_enabled(): return @@ -411,7 +508,7 @@ def _track_method(name: str, attrs: Optional[Dict[str, Any]] = None): @staticmethod def track_event( name: str, - properties: Optional[Dict[str, Any]] = None, + properties: Dict[str, Any] | None = None, ) -> None: """Track a custom event to Application Insights customEvents table. @@ -447,7 +544,7 @@ def track_event( def track_event( name: str, - properties: Optional[Dict[str, Any]] = None, + properties: Dict[str, Any] | None = None, ) -> None: """Track a custom event. @@ -488,7 +585,7 @@ def flush_events() -> None: def set_event_connection_string_provider( - provider: Callable[[], Optional[str]], + provider: Callable[[], str | None], ) -> None: """Override how the Application Insights connection string is resolved. @@ -505,7 +602,7 @@ def reset_event_client() -> None: def track_cli_event( name: str, - properties: Optional[Dict[str, Any]] = None, + properties: Dict[str, Any] | None = None, ) -> None: """Track a CLI event. @@ -521,10 +618,10 @@ def track_cli_event( def track( - name_or_func: Optional[Union[str, Callable[..., Any]]] = None, + name_or_func: str | Callable[..., Any] | None = None, *, - when: Optional[Union[bool, Callable[..., bool]]] = True, - extra: Optional[Dict[str, Any]] = None, + when: bool | Callable[..., bool] | None = True, + extra: Dict[str, Any] | None = None, ): """Decorator that will trace function invocations. diff --git a/packages/uipath/src/uipath/tracing/__init__.py b/packages/uipath/src/uipath/tracing/__init__.py index aaef6328c..a586e04b9 100644 --- a/packages/uipath/src/uipath/tracing/__init__.py +++ b/packages/uipath/src/uipath/tracing/__init__.py @@ -1,17 +1,23 @@ """Tracing utilities and OpenTelemetry exporters.""" from uipath.core import traced +from uipath.platform.common._reference_context import ( + ReferenceContext, + ReferenceContextAccessor, + ReferenceEntry, +) from uipath.platform.common._span_utils import ( AttachmentDirection, AttachmentProvider, SpanAttachment, + SpanStatus, + VerbosityLevel, ) from ._live_tracking_processor import LiveTrackingSpanProcessor from ._otel_exporters import ( # noqa: D104 JsonLinesFileExporter, LlmOpsHttpExporter, - SpanStatus, ) __all__ = [ @@ -23,4 +29,8 @@ "AttachmentDirection", "AttachmentProvider", "SpanAttachment", + "VerbosityLevel", + "ReferenceEntry", + "ReferenceContext", + "ReferenceContextAccessor", ] diff --git a/packages/uipath/src/uipath/tracing/_live_tracking_processor.py b/packages/uipath/src/uipath/tracing/_live_tracking_processor.py index 85bcca1ba..a338a87ed 100644 --- a/packages/uipath/src/uipath/tracing/_live_tracking_processor.py +++ b/packages/uipath/src/uipath/tracing/_live_tracking_processor.py @@ -5,7 +5,8 @@ from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor from uipath.core.tracing import UiPathTraceSettings -from uipath.tracing._otel_exporters import LlmOpsHttpExporter, SpanStatus +from uipath.platform.common._span_utils import SpanStatus +from uipath.tracing._otel_exporters import LlmOpsHttpExporter logger = logging.getLogger(__name__) @@ -46,7 +47,7 @@ def __init__( ) def _upsert_span_async( - self, span: Span | ReadableSpan, status_override: int | None = None + self, span: Span | ReadableSpan, status_override: SpanStatus | None = None ) -> None: """Run upsert_span in a background thread without blocking. @@ -57,7 +58,7 @@ def _upsert_span_async( def _upsert(): try: - if status_override: + if status_override is not None: self.exporter.upsert_span(span, status_override=status_override) else: self.exporter.upsert_span(span) @@ -88,6 +89,10 @@ def shutdown(self) -> None: self.executor.shutdown(wait=True) except Exception as e: logger.debug(f"Executor shutdown failed: {e}") + try: + self.exporter.shutdown() + except Exception as e: + logger.debug(f"Exporter shutdown failed: {e}") def force_flush(self, timeout_millis: int = 30000) -> bool: """Force flush - no-op for live tracking.""" diff --git a/packages/uipath/src/uipath/tracing/_otel_exporters.py b/packages/uipath/src/uipath/tracing/_otel_exporters.py index 423473065..6796c9629 100644 --- a/packages/uipath/src/uipath/tracing/_otel_exporters.py +++ b/packages/uipath/src/uipath/tracing/_otel_exporters.py @@ -13,7 +13,16 @@ from uipath._utils._ssl_context import get_httpx_client_kwargs from uipath.platform.common import _SpanUtils +from uipath.platform.common._span_utils import SpanSource, SpanStatus from uipath.platform.common.retry import NON_RETRYABLE_STATUS_CODES +from uipath.platform.constants import ( + ENV_BASE_URL, + ENV_ORGANIZATION_ID, + ENV_TENANT_ID, + ENV_UIPATH_ACCESS_TOKEN, + HEADER_INTERNAL_ACCOUNT_ID, + HEADER_INTERNAL_TENANT_ID, +) logger = logging.getLogger(__name__) @@ -24,17 +33,6 @@ def _normalize_process_key(value: Optional[str]) -> Optional[str]: return None if not value or value == _NIL_UUID else value -class SpanStatus: - """Span status values matching LLMOps StatusEnum.""" - - UNSET = 0 - OK = 1 - ERROR = 2 - RUNNING = 3 - RESTRICTED = 4 - CANCELLED = 5 - - def _safe_parse_json(s: Any) -> Any: """Safely parse a JSON string, returning the original if not a string or on error.""" if not isinstance(s, str): @@ -106,11 +104,6 @@ class LlmOpsHttpExporter(SpanExporter): # Add more mappings as needed } - class Status: - SUCCESS = 1 - ERROR = 2 - INTERRUPTED = 3 - def __init__( self, trace_id: Optional[str] = None, @@ -122,18 +115,16 @@ def __init__( """ super().__init__() self.base_url = self._get_base_url() - self.auth_token = os.environ.get("UIPATH_ACCESS_TOKEN") + self.auth_token = os.environ.get(ENV_UIPATH_ACCESS_TOKEN) self.headers: dict[str, str] = { "Content-Type": "application/json", "Authorization": f"Bearer {self.auth_token}", } if os.environ.get("UIPATH_TRACE_BASE_URL"): - self.headers["X-UiPath-Internal-TenantId"] = os.environ.get( - "UIPATH_TENANT_ID", "" - ) - self.headers["X-UiPath-Internal-AccountId"] = os.environ.get( - "UIPATH_ORGANIZATION_ID", "" + self.headers[HEADER_INTERNAL_TENANT_ID] = os.environ.get(ENV_TENANT_ID, "") + self.headers[HEADER_INTERNAL_ACCOUNT_ID] = os.environ.get( + ENV_ORGANIZATION_ID, "" ) client_kwargs = get_httpx_client_kwargs(headers=self.headers) @@ -148,7 +139,7 @@ def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: return SpanExportResult.SUCCESS logger.debug( - f"Exporting {len(spans)} spans to {self.base_url}/api/Traces/spans" + f"Exporting {len(spans)} spans to {self.base_url}/api/Traces/v3/spans" ) # Use optimized path: keep attributes as dict for processing @@ -185,10 +176,14 @@ def force_flush(self, timeout_millis: int = 30000) -> bool: """Force flush the exporter.""" return True + def shutdown(self) -> None: + """Close the HTTP client.""" + self.http_client.close() + def upsert_span( self, span: ReadableSpan, - status_override: Optional[int] = None, + status_override: Optional[SpanStatus] = None, ) -> SpanExportResult: """Upsert a single span to LLMOps for real-time state updates. @@ -312,12 +307,15 @@ def _map_tool_call_attributes(self, attributes: Dict[str, Any]) -> Dict[str, Any return result - def _determine_status(self, error: Optional[Any]) -> int: + def _determine_status(self, error: Optional[Any]) -> SpanStatus: if error: if isinstance(error, str) and error.startswith("GraphInterrupt("): - return self.Status.INTERRUPTED - return self.Status.ERROR - return self.Status.SUCCESS + # GraphInterrupt = HITL pause, not a failure. Server StatusEnum has + # no Interrupted member; preserves prior wire behavior (old int + # 3 == Running). + return SpanStatus.RUNNING + return SpanStatus.ERROR + return SpanStatus.OK def _process_span_attributes(self, span_data: Dict[str, Any]) -> None: """Extracts, transforms, and maps attributes for a span in-place. @@ -387,9 +385,15 @@ def _process_span_attributes(self, span_data: Dict[str, Any]) -> None: span_data["Status"] = status def _build_url(self, span_list: list[Dict[str, Any]]) -> str: - """Construct the URL for the API request.""" + """Construct the URL for the API request. + + The `source` query param is what the server persists as Trace.Source + (the span-body Source is ignored on ingest), so derive it from the + span's resolved SpanSource. Falls back to CodedAgents when absent. + """ trace_id = str(span_list[0]["TraceId"]) - return f"{self.base_url}/api/Traces/spans?traceId={trace_id}&source=Robots" + source = str(span_list[0].get("Source") or SpanSource.CODED_AGENTS) + return f"{self.base_url}/api/Traces/v3/spans?traceId={trace_id}&source={source}" def _send_with_retries( self, url: str, payload: list[Dict[str, Any]], max_retries: int = 4 @@ -421,7 +425,7 @@ def _get_base_url(self) -> str: return trace_base_url.rstrip("/") uipath_url = ( - os.environ.get("UIPATH_URL") + os.environ.get(ENV_BASE_URL) or "https://cloud.uipath.com/dummyOrg/dummyTennant/" ) diff --git a/packages/uipath/testcases/debug-simulation-testcase/pyproject.toml b/packages/uipath/testcases/debug-simulation-testcase/pyproject.toml new file mode 100644 index 000000000..e20c1b585 --- /dev/null +++ b/packages/uipath/testcases/debug-simulation-testcase/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "debug-simulation-testcase" +version = "0.0.1" +description = "debug-simulation-testcase" +authors = [{ name = "UiPath", email = "python-sdk@uipath.com" }] +dependencies = [ + "uipath", +] +requires-python = ">=3.11" + +[tool.uv.sources] +uipath = { path = "../../", editable = true } diff --git a/packages/uipath/testcases/debug-simulation-testcase/run.sh b/packages/uipath/testcases/debug-simulation-testcase/run.sh new file mode 100755 index 000000000..7880fa101 --- /dev/null +++ b/packages/uipath/testcases/debug-simulation-testcase/run.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -euo pipefail + +TESTCASE_DIR="$(cd "$(dirname "$0")" && pwd)" +SAMPLE_DIR="$(cd "$TESTCASE_DIR/../../samples/runtime-simulations-agent" && pwd)" + +echo "Syncing testcase dependencies (local editable uipath)..." +uv sync --project "$TESTCASE_DIR" + +UIPATH_BIN="$TESTCASE_DIR/.venv/bin/uipath" + +# Run auth and agent from the sample dir so credentials are stored and read +# from the same location. +cd "$SAMPLE_DIR" + +echo "Authenticating with UiPath..." +"$UIPATH_BIN" auth \ + --client-id="$CLIENT_ID" \ + --client-secret="$CLIENT_SECRET" \ + --base-url="$BASE_URL" + +echo "Running agent with debug + simulation..." +"$UIPATH_BIN" debug main \ + -f input.json \ + --attach none \ + --simulation "$(cat simulation.json)" 2>&1 | tee "$TESTCASE_DIR/run.log" + +# Copy the runtime output file back to the testcase dir for assert.py +mkdir -p "$TESTCASE_DIR/__uipath" +cp "$SAMPLE_DIR/__uipath/output.json" "$TESTCASE_DIR/__uipath/output.json" diff --git a/packages/uipath/testcases/debug-simulation-testcase/src/assert.py b/packages/uipath/testcases/debug-simulation-testcase/src/assert.py new file mode 100644 index 000000000..fd7e89697 --- /dev/null +++ b/packages/uipath/testcases/debug-simulation-testcase/src/assert.py @@ -0,0 +1,57 @@ +import json +import os + +# ── 1. Verify agent output exists and succeeded ────────────────────────────── +output_file = "__uipath/output.json" +assert os.path.isfile(output_file), "Agent output file not found" + +with open(output_file, "r", encoding="utf-8") as f: + output_data = json.load(f) + +status = output_data.get("status") +assert status == "successful", f"Agent execution failed with status: {status}" + +output = output_data.get("output", {}) + +assert "syntax" in output, "Missing 'syntax' in output" +assert "style" in output, "Missing 'style' in output" +assert "improvements" in output, "Missing 'improvements' in output" +assert "summary" in output, "Missing 'summary' in output" + +assert isinstance(output["syntax"]["valid"], bool), "'syntax.valid' must be a bool" +assert isinstance(output["syntax"]["errors"], list), "'syntax.errors' must be a list" + +score = output["style"]["score"] +assert isinstance(score, int), "'style.score' must be an int" +assert 0 <= score <= 100, f"'style.score' out of range: {score}" +assert isinstance(output["style"]["violations"], list), ( + "'style.violations' must be a list" +) + +assert isinstance(output["improvements"]["suggestions"], list), ( + "'improvements.suggestions' must be a list" +) +assert isinstance(output["improvements"]["refactored_snippet"], str), ( + "'improvements.refactored_snippet' must be a str" +) + +# ── 2. Verify simulation produced non-default values ───────────────────────── +# Real tool impls always return: score=100, violations=[], suggestions=[]. +# The LLM simulation should detect issues in the input code and return richer output. +simulated_something = ( + score < 100 + or len(output["style"]["violations"]) > 0 + or len(output["improvements"]["suggestions"]) > 0 +) +assert simulated_something, ( + "Output matches hardcoded real-tool defaults — simulation may not have run. " + f"style.score={score}, violations={output['style']['violations']}, " + f"suggestions={output['improvements']['suggestions']}" +) + +print( + f"Simulation confirmed: score={score}, " + f"violations={len(output['style']['violations'])}, " + f"suggestions={len(output['improvements']['suggestions'])}" +) +print("All assertions passed.") diff --git a/packages/uipath/testcases/langchain-cross/pyproject.toml b/packages/uipath/testcases/langchain-cross/pyproject.toml index ae2717777..58c3c38d2 100644 --- a/packages/uipath/testcases/langchain-cross/pyproject.toml +++ b/packages/uipath/testcases/langchain-cross/pyproject.toml @@ -3,11 +3,24 @@ name = "agent" version = "0.0.1" description = "agent" authors = [{ name = "John Doe", email = "john.doe@myemail.com" }] -dependencies = [ - "uipath-langchain>=0.0.149", - "uipath" -] +dependencies = [ + "uipath-langchain>=0.14.0,<0.15.0", + "uipath" +] requires-python = ">=3.11" -[tool.uv.sources] -uipath = { path = "../../", editable = true } \ No newline at end of file +[tool.uv.sources] +uipath = { path = "../../", editable = true } +uipath-core = { path = "../../../uipath-core", editable = true } +uipath-platform = { path = "../../../uipath-platform", editable = true } + +# Force local UiPath packages under test regardless of upper bounds declared by +# the published uipath-langchain package. Mirrors the uv override used in the +# cross-repo test workflows. +[tool.uv] +override-dependencies = [ + "uipath", + "uipath-core", + "uipath-platform", + "uipath-runtime>=0.13.0,<0.14.0", +] diff --git a/packages/uipath/testcases/list-target-output-key-evals/pyproject.toml b/packages/uipath/testcases/list-target-output-key-evals/pyproject.toml new file mode 100644 index 000000000..b3a04339d --- /dev/null +++ b/packages/uipath/testcases/list-target-output-key-evals/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "list-target-output-key-evals" +version = "0.0.1" +description = "Tests for evaluating multiple output keys at once using a list targetOutputKey" +authors = [{ name = "John Doe", email = "john.doe@myemail.com" }] +dependencies = [ + "uipath", +] +requires-python = ">=3.11" + +[tool.uv.sources] +uipath = { path = "../../", editable = true } diff --git a/packages/uipath/testcases/list-target-output-key-evals/run.sh b/packages/uipath/testcases/list-target-output-key-evals/run.sh new file mode 100755 index 000000000..508bd9777 --- /dev/null +++ b/packages/uipath/testcases/list-target-output-key-evals/run.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +echo "Syncing dependencies..." +uv sync + +echo "Authenticating with UiPath..." +uv run uipath auth --client-id="$CLIENT_ID" --client-secret="$CLIENT_SECRET" --base-url="$BASE_URL" + +echo "Running list targetOutputKey evaluations..." +uv run uipath eval main ../../samples/list_target_output_key_test/evaluations/eval-sets/default.json --no-report --output-file default.json + +echo "Test completed successfully!" diff --git a/packages/uipath/testcases/list-target-output-key-evals/src/assert.py b/packages/uipath/testcases/list-target-output-key-evals/src/assert.py new file mode 100644 index 000000000..80f22c195 --- /dev/null +++ b/packages/uipath/testcases/list-target-output-key-evals/src/assert.py @@ -0,0 +1,90 @@ +"""Assertions for list-target-output-key-evals testcase. + +Validates that evaluating multiple output fields at once (targetOutputKey as a +list) works correctly for both ExactMatch and JsonSimilarity evaluators. + +Expected outcomes +----------------- +- headphones-all-match : ListKeysExactMatch=1.0, ListKeysJsonSimilarity=1.0 +- shoes-all-match : ListKeysExactMatch=1.0, ListKeysJsonSimilarity=1.0 +- headphones-wrong-price: ListKeysExactMatch=0.0, ListKeysJsonSimilarity=1.0 +""" + +import json +import os + +# Maps evaluationName → evaluator ID → expected score (1.0 = pass, 0.0 = fail) +EXPECTED: dict[str, dict[str, float]] = { + "Headphones - all keys match": { + "ListKeysExactMatch": 1.0, + "ListKeysJsonSimilarity": 1.0, + }, + "Running Shoes - all keys match": { + "ListKeysExactMatch": 1.0, + "ListKeysJsonSimilarity": 1.0, + }, + "Headphones - wrong price (should fail)": { + "ListKeysExactMatch": 0.0, + "ListKeysJsonSimilarity": 1.0, + }, +} + + +def main() -> None: + output_file = "default.json" + assert os.path.isfile(output_file), f"Output file '{output_file}' not found" + print(f"Found output file: {output_file}") + + with open(output_file, "r", encoding="utf-8") as f: + output_data = json.load(f) + + assert "evaluationSetResults" in output_data, "Missing 'evaluationSetResults'" + evaluation_results = output_data["evaluationSetResults"] + assert len(evaluation_results) > 0, "No evaluation results found" + print(f"Found {len(evaluation_results)} evaluation result(s)") + + failures: list[str] = [] + + for eval_result in evaluation_results: + eval_name = eval_result.get("evaluationName", "") + expected_scores = EXPECTED.get(eval_name) + + if expected_scores is None: + print(f" [skip] '{eval_name}' not in EXPECTED map") + continue + + print(f"\n Validating: {eval_name}") + + run_results = eval_result.get("evaluationRunResults", []) + assert len(run_results) > 0, f"No run results for '{eval_name}'" + + for run in run_results: + evaluator_id = run.get("evaluatorId", run.get("evaluatorName", "")) + score = run.get("result", {}).get("score", None) + + if evaluator_id not in expected_scores: + print(f" [skip] unexpected evaluator '{evaluator_id}'") + continue + + expected = expected_scores[evaluator_id] + ok = score == expected + status = "pass" if ok else "FAIL" + print(f" {evaluator_id}: score={score} expected={expected} ({status})") + if not ok: + failures.append( + f"{eval_name} / {evaluator_id}: got {score}, expected {expected}" + ) + + print(f"\n{'=' * 60}") + if failures: + for f in failures: + print(f" FAIL: {f}") + print(f"{'=' * 60}") + assert False, f"{len(failures)} assertion(s) failed" + + print(" All assertions passed!") + print(f"{'=' * 60}") + + +if __name__ == "__main__": + main() diff --git a/packages/uipath/testcases/list-target-output-key-evals/uipath.json b/packages/uipath/testcases/list-target-output-key-evals/uipath.json new file mode 100644 index 000000000..a50f100df --- /dev/null +++ b/packages/uipath/testcases/list-target-output-key-evals/uipath.json @@ -0,0 +1,5 @@ +{ + "functions": { + "main": "../../samples/list_target_output_key_test/main.py:main" + } +} diff --git a/packages/uipath/testcases/simulation-testcase/pyproject.toml b/packages/uipath/testcases/simulation-testcase/pyproject.toml new file mode 100644 index 000000000..d37877dbf --- /dev/null +++ b/packages/uipath/testcases/simulation-testcase/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "simulation-testcase" +version = "0.0.1" +description = "simulation-testcase" +authors = [{ name = "UiPath", email = "python-sdk@uipath.com" }] +dependencies = [ + "uipath", +] +requires-python = ">=3.11" + +[tool.uv.sources] +uipath = { path = "../../", editable = true } diff --git a/packages/uipath/testcases/simulation-testcase/run.sh b/packages/uipath/testcases/simulation-testcase/run.sh new file mode 100644 index 000000000..0095cc904 --- /dev/null +++ b/packages/uipath/testcases/simulation-testcase/run.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -e + +TESTCASE_DIR="$(cd "$(dirname "$0")" && pwd)" +SAMPLE_DIR="$(cd "$TESTCASE_DIR/../../samples/runtime-simulations-agent" && pwd)" + +echo "Syncing testcase dependencies (local editable uipath)..." +uv sync --project "$TESTCASE_DIR" + +UIPATH_BIN="$TESTCASE_DIR/.venv/bin/uipath" + +# Run auth and agent from the sample dir so credentials are stored and read +# from the same location. +cd "$SAMPLE_DIR" + +echo "Authenticating with UiPath..." +"$UIPATH_BIN" auth \ + --client-id="$CLIENT_ID" \ + --client-secret="$CLIENT_SECRET" \ + --base-url="$BASE_URL" + +echo "Running agent with simulation..." +"$UIPATH_BIN" run main \ + -f input.json \ + --simulation "$(cat simulation.json)" 2>&1 | tee "$TESTCASE_DIR/run.log" + +# Copy the runtime output file back to the testcase dir for assert.py +mkdir -p "$TESTCASE_DIR/__uipath" +cp "$SAMPLE_DIR/__uipath/output.json" "$TESTCASE_DIR/__uipath/output.json" diff --git a/packages/uipath/testcases/simulation-testcase/src/assert.py b/packages/uipath/testcases/simulation-testcase/src/assert.py new file mode 100644 index 000000000..fd7e89697 --- /dev/null +++ b/packages/uipath/testcases/simulation-testcase/src/assert.py @@ -0,0 +1,57 @@ +import json +import os + +# ── 1. Verify agent output exists and succeeded ────────────────────────────── +output_file = "__uipath/output.json" +assert os.path.isfile(output_file), "Agent output file not found" + +with open(output_file, "r", encoding="utf-8") as f: + output_data = json.load(f) + +status = output_data.get("status") +assert status == "successful", f"Agent execution failed with status: {status}" + +output = output_data.get("output", {}) + +assert "syntax" in output, "Missing 'syntax' in output" +assert "style" in output, "Missing 'style' in output" +assert "improvements" in output, "Missing 'improvements' in output" +assert "summary" in output, "Missing 'summary' in output" + +assert isinstance(output["syntax"]["valid"], bool), "'syntax.valid' must be a bool" +assert isinstance(output["syntax"]["errors"], list), "'syntax.errors' must be a list" + +score = output["style"]["score"] +assert isinstance(score, int), "'style.score' must be an int" +assert 0 <= score <= 100, f"'style.score' out of range: {score}" +assert isinstance(output["style"]["violations"], list), ( + "'style.violations' must be a list" +) + +assert isinstance(output["improvements"]["suggestions"], list), ( + "'improvements.suggestions' must be a list" +) +assert isinstance(output["improvements"]["refactored_snippet"], str), ( + "'improvements.refactored_snippet' must be a str" +) + +# ── 2. Verify simulation produced non-default values ───────────────────────── +# Real tool impls always return: score=100, violations=[], suggestions=[]. +# The LLM simulation should detect issues in the input code and return richer output. +simulated_something = ( + score < 100 + or len(output["style"]["violations"]) > 0 + or len(output["improvements"]["suggestions"]) > 0 +) +assert simulated_something, ( + "Output matches hardcoded real-tool defaults — simulation may not have run. " + f"style.score={score}, violations={output['style']['violations']}, " + f"suggestions={output['improvements']['suggestions']}" +) + +print( + f"Simulation confirmed: score={score}, " + f"violations={len(output['style']['violations'])}, " + f"suggestions={len(output['improvements']['suggestions'])}" +) +print("All assertions passed.") diff --git a/packages/uipath/tests/agent/models/test_agent.py b/packages/uipath/tests/agent/models/test_agent.py index fe5c19bb9..4333dec38 100644 --- a/packages/uipath/tests/agent/models/test_agent.py +++ b/packages/uipath/tests/agent/models/test_agent.py @@ -1,13 +1,14 @@ from typing import Any import pytest -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from uipath.agent.models.agent import ( AgentA2aResourceConfig, AgentBooleanOperator, AgentBooleanRule, AgentBuiltInValidatorGuardrail, + AgentClientSideToolResourceConfig, AgentContextResourceConfig, AgentContextRetrievalMode, AgentContextType, @@ -26,6 +27,7 @@ AgentIntegrationToolResourceConfig, AgentInternalBatchTransformToolProperties, AgentInternalDeepRagToolProperties, + AgentInternalHttpRequestToolProperties, AgentInternalToolResourceConfig, AgentInternalToolType, AgentIxpExtractionResourceConfig, @@ -35,6 +37,7 @@ AgentNumberOperator, AgentNumberRule, AgentProcessToolResourceConfig, + AgentQuickFormChannelProperties, AgentResourceType, AgentToolArgumentPropertiesVariant, AgentToolType, @@ -43,16 +46,23 @@ AgentUnknownToolResourceConfig, AgentWordOperator, AgentWordRule, + ArgumentEmailRecipient, + ArgumentGroupNameRecipient, AssetRecipient, BatchTransformFileExtension, BatchTransformWebSearchGrounding, + CachedToolsConfig, CitationMode, + CustomAssigneesRecipient, DeepRagFileExtension, + RoundRobinRecipient, StandardRecipient, TaskTitleType, TextBuilderTaskTitle, TextToken, TextTokenType, + ToolOutputRecipient, + WorkloadRecipient, ) from uipath.platform.guardrails import ( EnumListParameterValue, @@ -723,6 +733,7 @@ def test_agent_config_loads_guardrails(self): == "This validator is designed to detect personally identifiable information using Azure Cognitive Services" ) assert agent_builtin_guardrail.enabled_for_evals is True + assert agent_builtin_guardrail.selector is not None assert agent_builtin_guardrail.selector.scopes == ["Tool"] assert agent_builtin_guardrail.selector.match_names == ["StringToNumber"] @@ -2029,6 +2040,53 @@ def test_mcp_resource_with_output_schema(self): assert tool2.output_schema is not None assert "content" in tool2.output_schema["properties"] + def test_cached_tools_config_refresh_schema_default(self): + """CachedToolsConfig defaults refresh_schema_before_call to True.""" + + config = CachedToolsConfig() + assert config.type == "cached" + assert config.refresh_schema_before_call is True + + def test_cached_tools_config_refresh_schema_alias_roundtrip(self): + """refresh_schema_before_call parses from and serializes to refreshSchemaBeforeCall.""" + + config = TypeAdapter(CachedToolsConfig).validate_python( + {"type": "cached", "refreshSchemaBeforeCall": False} + ) + assert config.refresh_schema_before_call is False + assert config.model_dump(by_alias=True)["refreshSchemaBeforeCall"] is False + + def test_mcp_resource_with_cached_tools_configuration(self): + """AgentMcpResourceConfig parses a cached toolsConfiguration with the refresh flag.""" + + json_data = { + "$resourceType": "mcp", + "folderPath": "solution_folder", + "slug": "tavily-mcp", + "name": "tavily", + "description": "Tavily search tools", + "isEnabled": True, + "availableTools": [ + { + "name": "tavily-search", + "description": "Search the web", + "inputSchema": {"type": "object", "properties": {}}, + } + ], + "toolsConfiguration": { + "discoveryMode": { + "type": "cached", + "refreshSchemaBeforeCall": False, + } + }, + } + + mcp_resource = TypeAdapter(AgentMcpResourceConfig).validate_python(json_data) + assert mcp_resource.tools_configuration is not None + discovery_mode = mcp_resource.tools_configuration.discovery_mode + assert isinstance(discovery_mode, CachedToolsConfig) + assert discovery_mode.refresh_schema_before_call is False + @pytest.mark.parametrize( "recipient_type_int,value,expected_type", [ @@ -2603,11 +2661,84 @@ def test_agent_with_ixp_vs_escalation(self): assert len(channel.recipients) == 0 # Validate channel properties + assert isinstance(channel, AgentEscalationChannel) assert channel.properties.app_name is None assert channel.properties.app_version == 1 assert channel.properties.folder_name is None assert channel.properties.resource_key is None + def test_quick_form_channel_properties_derive_schema_id_from_body(self): + """schema_id reads the schemaId nested inside the schema body.""" + + props = AgentQuickFormChannelProperties.model_validate( + { + "schema": { + "schemaId": "e74ebb74-80ba-47b9-a370-532a1ba4c41e", + "fields": [], + "outcomes": [], + }, + } + ) + assert props.schema_id == "e74ebb74-80ba-47b9-a370-532a1ba4c41e" + + def test_quick_form_channel_properties_schema_id_none_when_absent(self): + """schema_id is None when the schema body carries no schemaId.""" + + props = AgentQuickFormChannelProperties.model_validate( + {"schema": {"fields": [], "outcomes": []}} + ) + assert props.schema_id is None + + def test_quick_form_channel_properties_require_schema(self): + with pytest.raises(ValidationError): + AgentQuickFormChannelProperties.model_validate( + {"isActionableMessageEnabled": False} + ) + + def test_quick_form_channel_requires_schema(self): + """A quick-form channel without a schema fails to parse.""" + with pytest.raises(ValidationError): + TypeAdapter(AgentEscalationResourceConfig).validate_python( + { + "$resourceType": "escalation", + "name": "Escalation", + "description": "", + "channels": [ + { + "name": "c", + "description": "", + "inputSchema": {"type": "object", "properties": {}}, + "type": "actionCenterQuickForm", + "recipients": [], + "properties": {"isActionableMessageEnabled": False}, + } + ], + "isAgentMemoryEnabled": False, + } + ) + + def test_unknown_escalation_channel_type_is_rejected(self): + """An unrecognized channel type fails to parse; the runtime cannot handle it.""" + with pytest.raises(ValidationError): + TypeAdapter(AgentEscalationResourceConfig).validate_python( + { + "$resourceType": "escalation", + "name": "Escalation", + "description": "", + "channels": [ + { + "name": "c", + "description": "", + "inputSchema": {"type": "object", "properties": {}}, + "type": "someFutureChannel", + "recipients": [], + "properties": {}, + } + ], + "isAgentMemoryEnabled": False, + } + ) + def test_task_title_text_builder_type(self): """Test TextBuilderTaskTitle with tokens.""" from uipath.agent.models.agent import ( @@ -3027,6 +3158,24 @@ def test_case_insensitive_enums(self): "description": "Test batch transform tool", "isEnabled": True, }, + { + "$resourceType": "Tool", + "type": "Internal", + "inputSchema": { + "type": "object", + "properties": {}, + }, + "outputSchema": {"type": "object", "properties": {}}, + "arguments": {}, + "settings": {"timeout": 0, "maxAttempts": 0, "retryDelay": 0}, + "properties": { + "toolType": "Http-Request", + }, + "argumentProperties": {}, + "name": "HTTP Request Tool", + "description": "Test http request tool", + "isEnabled": True, + }, ], "guardrails": [ { @@ -3149,6 +3298,15 @@ def test_case_insensitive_enums(self): == BatchTransformWebSearchGrounding.ENABLED ) + http_request_tool = config.resources[5] + assert isinstance(http_request_tool, AgentInternalToolResourceConfig) + assert isinstance( + http_request_tool.properties, AgentInternalHttpRequestToolProperties + ) + assert ( + http_request_tool.properties.tool_type == AgentInternalToolType.HTTP_REQUEST + ) + assert config.guardrails is not None custom_guardrail = config.guardrails[0] assert isinstance(custom_guardrail, AgentCustomGuardrail) @@ -3280,6 +3438,119 @@ def test_is_conversational_false_by_default(self): assert config.is_conversational is False +class TestAgentDefinitionIsCaseManager: + """Tests for AgentDefinition.is_case_manager property.""" + + def test_is_case_manager_true_when_variant_is_case_manager(self): + """Returns True when metadata.variant is "caseManager".""" + json_data = { + "id": "test-case-manager", + "name": "Case Manager Agent", + "version": "1.0.0", + "metadata": { + "isConversational": False, + "variant": "caseManager", + "storageVersion": "1.0.0", + }, + "settings": { + "model": "gpt-4o", + "maxTokens": 4096, + "temperature": 0, + "engine": "basic-v1", + }, + "inputSchema": {"type": "object", "properties": {}}, + "outputSchema": {"type": "object", "properties": {}}, + "resources": [], + "messages": [ + {"role": "system", "content": "You are a case manager agent."} + ], + } + + config: AgentDefinition = TypeAdapter(AgentDefinition).validate_python( + json_data + ) + + assert config.is_case_manager is True + + def test_is_case_manager_false_when_variant_is_none(self): + """Returns False when metadata.variant is None.""" + json_data = { + "id": "test-non-case-manager", + "name": "Regular Agent", + "version": "1.0.0", + "metadata": { + "isConversational": False, + "variant": None, + "storageVersion": "1.0.0", + }, + "settings": { + "model": "gpt-4o", + "maxTokens": 4096, + "temperature": 0, + "engine": "basic-v1", + }, + "inputSchema": {"type": "object", "properties": {}}, + "outputSchema": {"type": "object", "properties": {}}, + "resources": [], + "messages": [{"role": "system", "content": "You are an agent."}], + } + + config: AgentDefinition = TypeAdapter(AgentDefinition).validate_python( + json_data + ) + + assert config.is_case_manager is False + + def test_is_case_manager_false_when_variant_not_in_metadata(self): + """Returns False when variant is not present in metadata.""" + json_data = { + "id": "test-no-variant-field", + "name": "Agent Without Variant Field", + "version": "1.0.0", + "metadata": {"isConversational": False, "storageVersion": "1.0.0"}, + "settings": { + "model": "gpt-4o", + "maxTokens": 4096, + "temperature": 0, + "engine": "basic-v1", + }, + "inputSchema": {"type": "object", "properties": {}}, + "outputSchema": {"type": "object", "properties": {}}, + "resources": [], + "messages": [{"role": "system", "content": "You are an agent."}], + } + + config: AgentDefinition = TypeAdapter(AgentDefinition).validate_python( + json_data + ) + + assert config.is_case_manager is False + + def test_is_case_manager_false_when_no_metadata(self): + """Returns False when agent has no metadata.""" + json_data = { + "id": "test-no-metadata", + "name": "Agent Without Metadata", + "version": "1.0.0", + "settings": { + "model": "gpt-4o", + "maxTokens": 4096, + "temperature": 0, + "engine": "basic-v1", + }, + "inputSchema": {"type": "object", "properties": {}}, + "outputSchema": {"type": "object", "properties": {}}, + "resources": [], + "messages": [{"role": "system", "content": "You are an agent."}], + } + + config: AgentDefinition = TypeAdapter(AgentDefinition).validate_python( + json_data + ) + + assert config.is_case_manager is False + + class TestAgentBuilderConfigResources: """Tests for AgentDefinition resource configuration parsing.""" @@ -3442,6 +3713,144 @@ def test_process_tool_missing_output_schema(self): assert isinstance(tool_resource, AgentProcessToolResourceConfig) assert tool_resource.output_schema == {"type": "object", "properties": {}} + def test_flow_tool_type_enum_value(self): + """AgentToolType.FLOW exists with the wire value 'Flow' and is case-insensitive.""" + assert AgentToolType.FLOW.value == "Flow" + assert AgentToolType("flow") is AgentToolType.FLOW + assert AgentToolType("FLOW") is AgentToolType.FLOW + + def test_flow_tool_resource_deserialization(self): + """A resource with type='Flow' is parsed as AgentProcessToolResourceConfig.""" + resources = [ + { + "$resourceType": "tool", + "type": "Flow", + "id": "flow-tool-1", + "inputSchema": { + "type": "object", + "properties": {"input": {"type": "string"}}, + }, + "outputSchema": {"type": "object", "properties": {}}, + "arguments": {}, + "settings": {"timeout": 0, "maxAttempts": 0, "retryDelay": 0}, + "properties": { + "processName": "MyFlow", + "folderPath": "/Shared/Flows", + }, + "name": "Flow Tool", + "description": "Test Flow tool", + } + ] + + json_data = self._agent_dict_with_resources(resources) + config: AgentDefinition = TypeAdapter(AgentDefinition).validate_python( + json_data + ) + + tool_resource = config.resources[0] + assert isinstance(tool_resource, AgentProcessToolResourceConfig) + assert tool_resource.type == AgentToolType.FLOW + assert tool_resource.properties.process_name == "MyFlow" + assert tool_resource.properties.folder_path == "/Shared/Flows" + + def test_flow_tool_resource_case_insensitive(self): + """A resource with lowercase type='flow' also deserializes via CaseInsensitiveEnum.""" + resources = [ + { + "$resourceType": "tool", + "type": "flow", + "id": "flow-tool-2", + "inputSchema": {"type": "object", "properties": {}}, + "outputSchema": {"type": "object", "properties": {}}, + "arguments": {}, + "settings": {"timeout": 0, "maxAttempts": 0, "retryDelay": 0}, + "properties": { + "processName": "MyFlow", + "folderPath": "/Shared/Flows", + }, + "name": "Flow Tool", + "description": "Test Flow tool", + } + ] + + json_data = self._agent_dict_with_resources(resources) + config: AgentDefinition = TypeAdapter(AgentDefinition).validate_python( + json_data + ) + + tool_resource = config.resources[0] + assert isinstance(tool_resource, AgentProcessToolResourceConfig) + assert tool_resource.type == AgentToolType.FLOW + + def test_function_tool_type_enum_value(self): + """AgentToolType.FUNCTION exists with the wire value 'Function' and is case-insensitive.""" + assert AgentToolType.FUNCTION.value == "Function" + assert AgentToolType("function") is AgentToolType.FUNCTION + assert AgentToolType("FUNCTION") is AgentToolType.FUNCTION + + def test_function_tool_resource_deserialization(self): + """A resource with type='Function' is parsed as AgentProcessToolResourceConfig.""" + resources = [ + { + "$resourceType": "tool", + "type": "Function", + "id": "function-tool-1", + "inputSchema": { + "type": "object", + "properties": {"input": {"type": "string"}}, + }, + "outputSchema": {"type": "object", "properties": {}}, + "arguments": {}, + "settings": {"timeout": 0, "maxAttempts": 0, "retryDelay": 0}, + "properties": { + "processName": "MyFunction", + "folderPath": "/Shared/Functions", + }, + "name": "Function Tool", + "description": "Test Function tool", + } + ] + + json_data = self._agent_dict_with_resources(resources) + config: AgentDefinition = TypeAdapter(AgentDefinition).validate_python( + json_data + ) + + tool_resource = config.resources[0] + assert isinstance(tool_resource, AgentProcessToolResourceConfig) + assert tool_resource.type == AgentToolType.FUNCTION + assert tool_resource.properties.process_name == "MyFunction" + assert tool_resource.properties.folder_path == "/Shared/Functions" + + def test_function_tool_resource_case_insensitive(self): + """A resource with lowercase type='function' also deserializes via CaseInsensitiveEnum.""" + resources = [ + { + "$resourceType": "tool", + "type": "function", + "id": "function-tool-2", + "inputSchema": {"type": "object", "properties": {}}, + "outputSchema": {"type": "object", "properties": {}}, + "arguments": {}, + "settings": {"timeout": 0, "maxAttempts": 0, "retryDelay": 0}, + "properties": { + "processName": "MyFunction", + "folderPath": "/Shared/Functions", + }, + "name": "Function Tool", + "description": "Test Function tool", + } + ] + + json_data = self._agent_dict_with_resources(resources) + config: AgentDefinition = TypeAdapter(AgentDefinition).validate_python( + json_data + ) + + tool_resource = config.resources[0] + assert isinstance(tool_resource, AgentProcessToolResourceConfig) + assert tool_resource.type == AgentToolType.FUNCTION + def test_escalation_missing_escalation_type_defaults_to_zero(self): """Test that missing escalationType defaults to 0.""" resources = [ @@ -3528,12 +3937,133 @@ def test_datafabric_context_config_parses(self): assert len(parsed.entity_set) == 2 assert parsed.entity_set[0].id == "abc-123" assert parsed.entity_set[0].name == "Customers" - assert parsed.entity_set[0].folder_id == "folder-1" + assert parsed.entity_set[0].folder_key == "folder-1" assert parsed.entity_set[0].description == "Customer records" - assert parsed.entity_set[0].reference_key is None - assert parsed.entity_set[1].reference_key == "orders-ref" + assert parsed.entity_set[0].entity_key is None + assert parsed.entity_set[1].entity_key == "orders-ref" assert parsed.entity_set[1].description is None + def test_ontology_context_parses(self): + """The ontology context (datafabricontology) holds an ontologySet array.""" + config = { + "$resourceType": "context", + "name": "Ontologies", + "description": "", + "contextType": "datafabricontology", + "ontologySet": [ + {"name": "library", "folderId": "f1"}, + {"name": "finance", "folderId": "f2"}, + ], + } + + parsed = AgentContextResourceConfig.model_validate(config) + + assert parsed.is_datafabric_ontology + assert not parsed.is_datafabric + assert parsed.ontology_set is not None + assert len(parsed.ontology_set) == 2 + assert parsed.ontology_set[0].name == "library" + assert parsed.ontology_set[0].folder_key == "f1" + assert parsed.ontology_set[1].name == "finance" + + def test_ontology_item_requires_folder_id(self): + """folderId is required on each ontology item.""" + config = { + "$resourceType": "context", + "name": "Ontologies", + "description": "", + "contextType": "datafabricontology", + "ontologySet": [{"name": "library"}], # missing folderId + } + + with pytest.raises(ValidationError): + AgentContextResourceConfig.model_validate(config) + + def test_ontology_context_dumps_by_alias(self): + """The ontology context round-trips back to aliased JSON keys.""" + parsed = AgentContextResourceConfig.model_validate( + { + "$resourceType": "context", + "name": "Ontologies", + "description": "", + "contextType": "datafabricontology", + "ontologySet": [{"name": "library", "folderId": "f1"}], + } + ) + dumped = parsed.model_dump(by_alias=True, exclude_none=True) + + assert dumped["contextType"] == "datafabricontology" + assert dumped["ontologySet"][0]["name"] == "library" + assert dumped["ontologySet"][0]["folderId"] == "f1" + + def test_entity_context_has_no_ontology_set(self): + """A plain entity context has no ontologySet and is not an ontology context.""" + config = { + "$resourceType": "context", + "name": "TestDataFabric", + "description": "", + "contextType": "datafabricentityset", + "entitySet": [{"id": "e1", "name": "Customers", "folderId": "f1"}], + } + + parsed = AgentContextResourceConfig.model_validate(config) + + assert parsed.is_datafabric + assert not parsed.is_datafabric_ontology + assert parsed.ontology_set is None + + def test_ontology_context_survives_full_definition_normalization(self): + """Regression: the datafabricontology context (with its ontologySet) + survives the full AgentDefinition normalizer and sits beside the entity + context, so the runtime can gather its ontologies to ground the DF query. + """ + json_data = { + "id": "test-ontology-def", + "name": "Agent with ontology context", + "version": "1.0.0", + "settings": { + "model": "gpt-4o-2024-11-20", + "maxTokens": 16384, + "temperature": 0, + "engine": "basic-v1", + }, + "inputSchema": {"type": "object", "properties": {}}, + "outputSchema": {"type": "object", "properties": {}}, + "resources": [ + { + "$resourceType": "context", + "contextType": "datafabricentityset", + "name": "Entities", + "description": "DF context", + "entitySet": [ + {"id": "e1", "name": "LibraryLoan", "folderId": "f1"} + ], + }, + { + "$resourceType": "context", + "contextType": "datafabricontology", + "name": "Ontologies", + "description": "", + "ontologySet": [{"name": "library", "folderId": "f1"}], + }, + ], + "messages": [{"role": "system", "content": "Test system message"}], + } + + config: AgentDefinition = TypeAdapter(AgentDefinition).validate_python( + json_data + ) + + ontology_ctxs = [ + r + for r in config.resources + if isinstance(r, AgentContextResourceConfig) and r.is_datafabric_ontology + ] + assert len(ontology_ctxs) == 1 + assert ontology_ctxs[0].ontology_set is not None + assert ontology_ctxs[0].ontology_set[0].name == "library" + assert ontology_ctxs[0].ontology_set[0].folder_key == "f1" + def test_is_datafabric(self): """Test is_datafabric property with datafabricentityset contextType.""" config = { @@ -3613,8 +4143,7 @@ def test_a2a_resource(self): "name": "Philosopher Agent", "slug": "philosopher-agent", "description": "A philosophical agent that answers questions with wisdom and philosopher quotes", - "agentCardUrl": "", - "isActive": True, + "folderPath": "shared", "cachedAgentCard": { "name": "Philosopher Agent", "description": "Philosopher Agent assistant", @@ -3665,10 +4194,6 @@ def test_a2a_resource(self): ], "version": "0.7.70", }, - "createdAt": "2026-03-15T10:12:47.9073065", - "createdBy": "f4bc4946-baed-4083-82b9-03d334bbacbe", - "updatedAt": None, - "updatedBy": None, } ], "features": [], @@ -3692,13 +4217,8 @@ def test_a2a_resource(self): a2a_resource.description == "A philosophical agent that answers questions with wisdom and philosopher quotes" ) - assert a2a_resource.is_active is True - assert a2a_resource.agent_card_url == "" assert a2a_resource.id == "755e2f7d-5a3d-47f3-8e9d-7ff0bf226357" - assert a2a_resource.created_at == "2026-03-15T10:12:47.9073065" - assert a2a_resource.created_by == "f4bc4946-baed-4083-82b9-03d334bbacbe" - assert a2a_resource.updated_at is None - assert a2a_resource.updated_by is None + assert a2a_resource.folder_path == "shared" # Validate cached agent card is a plain dict card = a2a_resource.cached_agent_card @@ -3736,7 +4256,7 @@ def test_a2a_resource_without_cached_card(self): "name": "Minimal A2A Agent", "slug": "minimal-a2a", "description": "A minimal A2A agent", - "isActive": False, + "folderPath": "shared", } ], "features": [], @@ -3755,10 +4275,8 @@ def test_a2a_resource_without_cached_card(self): assert isinstance(a2a_resource, AgentA2aResourceConfig) assert a2a_resource.name == "Minimal A2A Agent" assert a2a_resource.slug == "minimal-a2a" - assert a2a_resource.is_active is False + assert a2a_resource.folder_path == "shared" assert a2a_resource.cached_agent_card is None - assert a2a_resource.agent_card_url == "" - assert a2a_resource.created_at is None def test_a2a_resource_case_insensitive(self): """Test that A2A resource type is parsed case-insensitively.""" @@ -3784,6 +4302,7 @@ def test_a2a_resource_case_insensitive(self): "name": "Case Test Agent", "slug": "case-test", "description": "Testing case insensitive parsing", + "folderPath": "shared", } ], "features": [], @@ -3799,3 +4318,435 @@ def test_a2a_resource_case_insensitive(self): ] assert len(a2a_resources) == 1 assert isinstance(a2a_resources[0], AgentA2aResourceConfig) + + +class TestArgumentRecipientDeserialization: + def test_argument_email_recipient_by_type_int(self): + payload = {"type": 7, "argumentName": "assigneeEmail"} + recipient: AgentEscalationRecipient = TypeAdapter( + AgentEscalationRecipient + ).validate_python(payload) + assert isinstance(recipient, ArgumentEmailRecipient) + assert recipient.argument_path == "assigneeEmail" + assert recipient.type == AgentEscalationRecipientType.ARGUMENT_EMAIL + + def test_argument_group_name_recipient_by_type_int(self): + payload = {"type": 8, "argumentName": "assigneeGroup"} + recipient: AgentEscalationRecipient = TypeAdapter( + AgentEscalationRecipient + ).validate_python(payload) + assert isinstance(recipient, ArgumentGroupNameRecipient) + assert recipient.argument_path == "assigneeGroup" + assert recipient.type == AgentEscalationRecipientType.ARGUMENT_GROUP_NAME + + def test_argument_email_recipient_by_type_string(self): + payload = {"type": "ArgumentEmail", "argumentName": "emailArg"} + recipient: AgentEscalationRecipient = TypeAdapter( + AgentEscalationRecipient + ).validate_python(payload) + assert isinstance(recipient, ArgumentEmailRecipient) + assert recipient.argument_path == "emailArg" + + def test_argument_group_name_recipient_by_type_string(self): + payload = {"type": "ArgumentGroupName", "argumentName": "groupArg"} + recipient: AgentEscalationRecipient = TypeAdapter( + AgentEscalationRecipient + ).validate_python(payload) + assert isinstance(recipient, ArgumentGroupNameRecipient) + assert recipient.argument_path == "groupArg" + + def test_argument_email_recipient_missing_argument_name_raises(self): + payload = {"type": 7} + with pytest.raises(ValidationError): + TypeAdapter(AgentEscalationRecipient).validate_python(payload) + + def test_argument_group_name_recipient_missing_argument_name_raises(self): + payload = {"type": 8} + with pytest.raises(ValidationError): + TypeAdapter(AgentEscalationRecipient).validate_python(payload) + + def test_agent_with_client_side_tool(self): + """Test agent with ClientSide tool resource.""" + + json_data = { + "version": "1.0.0", + "id": "aaaaaaaa-0000-0000-0000-000000000010", + "name": "Agent with ClientSide Tool", + "metadata": {"isConversational": False, "storageVersion": "26.0.0"}, + "messages": [ + {"role": "System", "content": "You are an agentic assistant."}, + ], + "inputSchema": {"type": "object", "properties": {}}, + "outputSchema": { + "type": "object", + "properties": {"content": {"type": "string"}}, + }, + "settings": { + "model": "gpt-4o-2024-11-20", + "maxTokens": 16384, + "temperature": 0, + "engine": "basic-v2", + }, + "resources": [ + { + "$resourceType": "tool", + "id": "cst-0001-0000-0000-000000000001", + "name": "browser_navigate", + "description": "Navigate to a URL in the browser", + "location": "external", + "type": "ClientSide", + "inputSchema": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The URL to navigate to", + } + }, + "required": ["url"], + }, + "outputSchema": { + "type": "object", + "properties": { + "title": {"type": "string"}, + "content": {"type": "string"}, + }, + }, + "arguments": {"timeout": 30}, + "properties": {}, + "isEnabled": True, + } + ], + "features": [], + } + + config: AgentDefinition = TypeAdapter(AgentDefinition).validate_python( + json_data + ) + + assert config.name == "Agent with ClientSide Tool" + assert len(config.resources) == 1 + + tool = config.resources[0] + assert isinstance(tool, AgentClientSideToolResourceConfig) + assert tool.resource_type == AgentResourceType.TOOL + assert tool.type == AgentToolType.CLIENT_SIDE + assert tool.name == "browser_navigate" + assert tool.description == "Navigate to a URL in the browser" + + # Validate input schema + assert tool.input_schema["type"] == "object" + assert "url" in tool.input_schema["properties"] + assert tool.input_schema["required"] == ["url"] + + # Validate outputSchema alias deserializes to output_schema + assert tool.output_schema is not None + assert tool.output_schema["type"] == "object" + assert "title" in tool.output_schema["properties"] + assert "content" in tool.output_schema["properties"] + + # Validate arguments + assert tool.arguments == {"timeout": 30} + + def test_agent_with_client_side_tool_lowercase_type(self): + """Test that _normalize_resources handles lowercase 'clientside' type.""" + + json_data = { + "version": "1.0.0", + "id": "aaaaaaaa-0000-0000-0000-000000000011", + "name": "Agent with clientside Tool", + "metadata": {"isConversational": False, "storageVersion": "26.0.0"}, + "messages": [ + {"role": "System", "content": "You are an agentic assistant."}, + ], + "inputSchema": {"type": "object", "properties": {}}, + "outputSchema": { + "type": "object", + "properties": {"content": {"type": "string"}}, + }, + "settings": { + "model": "gpt-4o-2024-11-20", + "maxTokens": 16384, + "temperature": 0, + "engine": "basic-v2", + }, + "resources": [ + { + "$resourceType": "tool", + "id": "cst-0002-0000-0000-000000000001", + "name": "clipboard_copy", + "description": "Copy text to clipboard", + "location": "external", + "type": "clientside", + "inputSchema": { + "type": "object", + "properties": { + "text": {"type": "string"}, + }, + }, + "properties": {}, + "isEnabled": True, + } + ], + "features": [], + } + + config: AgentDefinition = TypeAdapter(AgentDefinition).validate_python( + json_data + ) + + tool = config.resources[0] + assert isinstance(tool, AgentClientSideToolResourceConfig) + assert tool.type == AgentToolType.CLIENT_SIDE + assert tool.name == "clipboard_copy" + + # output_schema and arguments should default + assert tool.output_schema is None + assert tool.arguments == {} + + def test_agent_with_client_side_tool_output_schema_alias(self): + """Test that the outputSchema alias correctly maps to output_schema.""" + + json_data = { + "version": "1.0.0", + "id": "aaaaaaaa-0000-0000-0000-000000000012", + "name": "Agent with ClientSide outputSchema alias", + "metadata": {"isConversational": False, "storageVersion": "26.0.0"}, + "messages": [ + {"role": "System", "content": "You are an agentic assistant."}, + ], + "inputSchema": {"type": "object", "properties": {}}, + "outputSchema": { + "type": "object", + "properties": {"content": {"type": "string"}}, + }, + "settings": { + "model": "gpt-4o-2024-11-20", + "maxTokens": 16384, + "temperature": 0, + "engine": "basic-v2", + }, + "resources": [ + { + "$resourceType": "tool", + "id": "cst-0003-0000-0000-000000000001", + "name": "screen_capture", + "description": "Capture a screenshot", + "location": "external", + "type": "ClientSide", + "inputSchema": { + "type": "object", + "properties": { + "region": {"type": "string"}, + }, + }, + "outputSchema": { + "type": "object", + "properties": { + "imageBase64": { + "type": "string", + "description": "Base64-encoded image", + } + }, + "required": ["imageBase64"], + }, + "properties": {}, + "isEnabled": True, + } + ], + "features": [], + } + + config: AgentDefinition = TypeAdapter(AgentDefinition).validate_python( + json_data + ) + + tool = config.resources[0] + assert isinstance(tool, AgentClientSideToolResourceConfig) + + # Access via Python attribute name (snake_case) + assert tool.output_schema is not None + assert tool.output_schema["type"] == "object" + assert "imageBase64" in tool.output_schema["properties"] + assert tool.output_schema["required"] == ["imageBase64"] + + +class TestCustomAssignmentRecipientDeserialization: + def test_workload_recipient_by_type_int(self): + payload = {"type": 9, "value": "group-1", "displayName": "Support Team"} + recipient: AgentEscalationRecipient = TypeAdapter( + AgentEscalationRecipient + ).validate_python(payload) + assert isinstance(recipient, WorkloadRecipient) + assert recipient.value == "group-1" + assert recipient.display_name == "Support Team" + assert recipient.type == AgentEscalationRecipientType.WORKLOAD + + def test_workload_recipient_by_type_string(self): + payload = { + "type": "Workload", + "value": "group-1", + "displayName": "Support Team", + } + recipient: AgentEscalationRecipient = TypeAdapter( + AgentEscalationRecipient + ).validate_python(payload) + assert isinstance(recipient, WorkloadRecipient) + assert recipient.value == "group-1" + assert recipient.display_name == "Support Team" + + def test_round_robin_recipient_by_type_int(self): + payload = {"type": 10, "value": "group-1", "displayName": "Support Team"} + recipient: AgentEscalationRecipient = TypeAdapter( + AgentEscalationRecipient + ).validate_python(payload) + assert isinstance(recipient, RoundRobinRecipient) + assert recipient.value == "group-1" + assert recipient.display_name == "Support Team" + assert recipient.type == AgentEscalationRecipientType.ROUND_ROBIN + + def test_round_robin_recipient_by_type_string(self): + payload = { + "type": "RoundRobin", + "value": "group-1", + "displayName": "Support Team", + } + recipient: AgentEscalationRecipient = TypeAdapter( + AgentEscalationRecipient + ).validate_python(payload) + assert isinstance(recipient, RoundRobinRecipient) + + def test_custom_assignees_recipient_by_type_int(self): + payload = { + "type": 11, + "value": "alice@example.com", + "displayName": "Alice", + } + recipient: AgentEscalationRecipient = TypeAdapter( + AgentEscalationRecipient + ).validate_python(payload) + assert isinstance(recipient, CustomAssigneesRecipient) + assert recipient.value == "alice@example.com" + assert recipient.display_name == "Alice" + assert recipient.type == AgentEscalationRecipientType.CUSTOM_ASSIGNEES + + def test_custom_assignees_recipient_by_type_string(self): + payload = {"type": "CustomAssignees", "value": "alice@example.com"} + recipient: AgentEscalationRecipient = TypeAdapter( + AgentEscalationRecipient + ).validate_python(payload) + assert isinstance(recipient, CustomAssigneesRecipient) + assert recipient.value == "alice@example.com" + assert recipient.display_name is None + + def test_custom_assignees_recipient_accepts_empty_value_sentinel(self): + payload = {"type": 11, "value": ""} + recipient: AgentEscalationRecipient = TypeAdapter( + AgentEscalationRecipient + ).validate_python(payload) + assert isinstance(recipient, CustomAssigneesRecipient) + assert recipient.value == "" + + def test_workload_recipient_missing_value_raises(self): + payload = {"type": 9, "displayName": "Support Team"} + with pytest.raises(ValidationError): + TypeAdapter(AgentEscalationRecipient).validate_python(payload) + + def test_workload_recipient_missing_display_name_raises(self): + payload = {"type": 9, "value": "group-1"} + with pytest.raises(ValidationError): + TypeAdapter(AgentEscalationRecipient).validate_python(payload) + + def test_round_robin_recipient_missing_value_raises(self): + payload = {"type": 10, "displayName": "Support Team"} + with pytest.raises(ValidationError): + TypeAdapter(AgentEscalationRecipient).validate_python(payload) + + def test_custom_assignees_recipient_missing_value_raises(self): + payload = {"type": 11} + with pytest.raises(ValidationError): + TypeAdapter(AgentEscalationRecipient).validate_python(payload) + + +class TestToolOutputRecipientDeserialization: + @pytest.mark.parametrize( + "recipient_type", + [1, 2, 9, 10, 11], + ) + def test_tool_output_recipient_by_type_int_for_supported_types( + self, recipient_type + ): + payload = { + "type": recipient_type, + "source": "toolOutput", + "toolName": "API workflow A", + "outputPath": "includeEmails", + } + recipient: AgentEscalationRecipient = TypeAdapter( + AgentEscalationRecipient + ).validate_python(payload) + assert isinstance(recipient, ToolOutputRecipient) + assert recipient.tool_name == "API workflow A" + assert recipient.output_path == "includeEmails" + assert recipient.source == "toolOutput" + + def test_tool_output_recipient_for_custom_assignees_by_type_string(self): + payload = { + "type": "CustomAssignees", + "source": "toolOutput", + "toolName": "API workflow A", + "outputPath": "includeEmails", + } + recipient: AgentEscalationRecipient = TypeAdapter( + AgentEscalationRecipient + ).validate_python(payload) + assert isinstance(recipient, ToolOutputRecipient) + assert recipient.type == AgentEscalationRecipientType.CUSTOM_ASSIGNEES + + def test_tool_output_recipient_missing_tool_name_raises(self): + payload = {"type": 11, "source": "toolOutput", "outputPath": "emails"} + with pytest.raises(ValidationError): + TypeAdapter(AgentEscalationRecipient).validate_python(payload) + + def test_tool_output_recipient_missing_output_path_raises(self): + payload = {"type": 11, "source": "toolOutput", "toolName": "A"} + with pytest.raises(ValidationError): + TypeAdapter(AgentEscalationRecipient).validate_python(payload) + + def test_tool_output_recipient_unknown_source_raises(self): + payload = { + "type": 11, + "source": "magicBox", + "toolName": "A", + "outputPath": "emails", + } + with pytest.raises(ValidationError): + TypeAdapter(AgentEscalationRecipient).validate_python(payload) + + @pytest.mark.parametrize( + "recipient_type", + [3, 4, 5, 6, 7, 8], + ) + def test_tool_output_recipient_not_allowed_for_static_asset_argument_types( + self, recipient_type + ): + # Static/asset/argument types (3, 4, 5, 6, 7, 8) are not supported + # for tool-output binding because they have their own design-time + # resolution rules. + payload = { + "type": recipient_type, + "source": "toolOutput", + "toolName": "A", + "outputPath": "emails", + } + with pytest.raises(ValidationError): + TypeAdapter(AgentEscalationRecipient).validate_python(payload) + + def test_literal_recipient_without_source_still_parses_to_literal_class(self): + # Backward compat: a payload without `source` still matches the literal class. + payload = {"type": 11, "value": "alice@example.com", "displayName": "Alice"} + recipient: AgentEscalationRecipient = TypeAdapter( + AgentEscalationRecipient + ).validate_python(payload) + assert isinstance(recipient, CustomAssigneesRecipient) + assert not isinstance(recipient, ToolOutputRecipient) diff --git a/packages/uipath/tests/agent/react/test_conversational_prompts.py b/packages/uipath/tests/agent/react/test_conversational_prompts.py index a58a94807..2ddafc904 100644 --- a/packages/uipath/tests/agent/react/test_conversational_prompts.py +++ b/packages/uipath/tests/agent/react/test_conversational_prompts.py @@ -8,6 +8,7 @@ from uipath.agent.react.conversational_prompts import ( PromptUserSettings, get_chat_system_prompt, + get_generate_output_prompt, get_user_settings_template, ) @@ -149,6 +150,68 @@ def test_generate_system_prompt_unnamed_agent_uses_default(self): assert "You are Unnamed Agent." in prompt +class TestConversationIdInPrompt: + """Tests for conversation_id in generated prompts.""" + + def test_prompt_includes_conversation_id_when_provided(self): + prompt = get_chat_system_prompt( + model="claude-3-sonnet", + system_message=SYSTEM_MESSAGE, + agent_name="Test Agent", + user_settings=None, + conversation_id="conv-abc-123", + ) + + assert "The current conversation ID is conv-abc-123" in prompt + assert ( + "This may be useful to include in tool-calls when tool parameters specify passing in the conversation ID." + in prompt + ) + assert ( + "{{CONVERSATIONAL_AGENT_SERVICE_PREFIX_conversationIdPrompt}}" not in prompt + ) + + def test_prompt_omits_section_when_none(self): + prompt = get_chat_system_prompt( + model="claude-3-sonnet", + system_message=SYSTEM_MESSAGE, + agent_name="Test Agent", + user_settings=None, + conversation_id=None, + ) + + assert "conversation ID" not in prompt + assert ( + "{{CONVERSATIONAL_AGENT_SERVICE_PREFIX_conversationIdPrompt}}" not in prompt + ) + + def test_prompt_omits_section_when_empty_string(self): + prompt = get_chat_system_prompt( + model="claude-3-sonnet", + system_message=SYSTEM_MESSAGE, + agent_name="Test Agent", + user_settings=None, + conversation_id="", + ) + + assert "conversation ID" not in prompt + + def test_prompt_defaults_to_no_conversation_id(self): + """conversation_id defaults to None — call sites that don't pass it + must not get a dangling placeholder.""" + prompt = get_chat_system_prompt( + model="claude-3-sonnet", + system_message=SYSTEM_MESSAGE, + agent_name="Test Agent", + user_settings=None, + ) + + assert ( + "{{CONVERSATIONAL_AGENT_SERVICE_PREFIX_conversationIdPrompt}}" not in prompt + ) + assert "conversation ID" not in prompt + + class TestCitationFormat: """Tests for citation format in generated prompts.""" @@ -286,3 +349,16 @@ def test_full_settings_json_format(self): assert json_data["company"] == "Big Corp" assert json_data["country"] == "UK" assert json_data["timezone"] == "Europe/London" + + +class TestGetGenerateOutputPrompt: + """Tests for get_generate_output_prompt function.""" + + def test_returns_non_empty_string(self): + instruction = get_generate_output_prompt() + assert isinstance(instruction, str) + assert instruction.strip() + + def test_references_set_conversational_output_tool(self): + """The instruction must name the tool the new node binds.""" + assert "set_conversational_output" in get_generate_output_prompt() diff --git a/packages/uipath/tests/cli/_governance/__init__.py b/packages/uipath/tests/cli/_governance/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/uipath/tests/cli/_governance/test_yaml_index.py b/packages/uipath/tests/cli/_governance/test_yaml_index.py new file mode 100644 index 000000000..6785a15a8 --- /dev/null +++ b/packages/uipath/tests/cli/_governance/test_yaml_index.py @@ -0,0 +1,775 @@ +"""Tests for ``build_policy_index_from_yaml``.""" + +from __future__ import annotations + +import pytest + +from uipath._cli._governance.yaml_index import ( + build_policy_index_from_yaml, +) +from uipath.core.governance.models import Action, LifecycleHook +from uipath.runtime.governance.native.models import Severity + + +def _single_rule(yaml_text: str): + """Compile YAML and return the single rule; fail if not exactly one.""" + idx = build_policy_index_from_yaml(yaml_text) + rules = idx.all_rules + assert len(rules) == 1, f"expected 1 rule, got {len(rules)}" + return rules[0] + + +def test_empty_yaml_returns_empty_index() -> None: + idx = build_policy_index_from_yaml("") + assert idx.total_rules == 0 + assert idx.pack_names == [] + + +def test_pack_without_rules_is_omitted() -> None: + """Packs with no parseable rules are dropped — never registered.""" + idx = build_policy_index_from_yaml( + """ + standard: empty-pack + version: "1.0" + rules: [] + """ + ) + assert idx.total_rules == 0 + assert "empty-pack" not in idx.pack_names + + +def test_pack_missing_name_is_skipped() -> None: + idx = build_policy_index_from_yaml( + """ + version: "1.0" + rules: + - id: r1 + hook: before_model + checks: + - type: regex + patterns: ["foo"] + """ + ) + assert idx.total_rules == 0 + + +def test_pack_uses_standard_or_name_field() -> None: + """Either ``standard:`` or ``name:`` works as the pack identifier.""" + a = build_policy_index_from_yaml( + """ + standard: iso42001 + rules: + - id: r + hook: before_model + checks: [{type: regex, patterns: ["x"]}] + """ + ) + b = build_policy_index_from_yaml( + """ + name: iso42001 + rules: + - id: r + hook: before_model + checks: [{type: regex, patterns: ["x"]}] + """ + ) + assert "iso42001" in a.pack_names + assert "iso42001" in b.pack_names + + +def test_multi_document_yaml_concatenates_packs() -> None: + # YAML doc separators must be at column 0; dedent inline. + yaml_text = ( + "standard: pack-a\n" + "rules:\n" + " - id: a-r1\n" + " hook: before_model\n" + ' checks: [{type: regex, patterns: ["a"]}]\n' + "---\n" + "standard: pack-b\n" + "rules:\n" + " - id: b-r1\n" + " hook: after_model\n" + ' checks: [{type: regex, patterns: ["b"]}]\n' + ) + idx = build_policy_index_from_yaml(yaml_text) + assert set(idx.pack_names) == {"pack-a", "pack-b"} + assert idx.total_rules == 2 + + +def test_non_dict_top_level_documents_are_ignored() -> None: + """A YAML doc that's a string / list at top level is skipped silently.""" + yaml_text = ( + "just_a_string\n" + "---\n" + "standard: real-pack\n" + "rules:\n" + " - id: r\n" + " hook: before_model\n" + ' checks: [{type: regex, patterns: ["x"]}]\n' + ) + idx = build_policy_index_from_yaml(yaml_text) + assert idx.pack_names == ["real-pack"] + + +def test_unknown_hook_skips_rule() -> None: + """A rule referencing an unknown hook is dropped, the rest survive.""" + idx = build_policy_index_from_yaml( + """ + standard: p + rules: + - id: bad + hook: invented_hook + checks: [{type: regex, patterns: ["x"]}] + - id: good + hook: before_model + checks: [{type: regex, patterns: ["x"]}] + """ + ) + rule_ids = [r.rule_id for r in idx.all_rules] + assert "bad" not in rule_ids + assert "good" in rule_ids + + +def test_non_dict_rule_entry_ignored() -> None: + """Rules entries that aren't dicts (lists, scalars) are skipped.""" + idx = build_policy_index_from_yaml( + """ + standard: p + rules: + - "this is a string, not a rule" + - id: good + hook: before_model + checks: [{type: regex, patterns: ["x"]}] + """ + ) + assert [r.rule_id for r in idx.all_rules] == ["good"] + + +def test_action_resolution_inherits_pack_default() -> None: + """When the rule omits action, the pack's default_action is used.""" + rule = _single_rule( + """ + standard: p + default_action: log + rules: + - id: r + hook: before_model + checks: [{type: regex, patterns: ["x"]}] + """ + ) + assert rule.action == Action.AUDIT # log -> AUDIT per _ACTION_MAP + + +def test_action_resolution_unknown_falls_back_to_default() -> None: + """Unknown action string falls back to the pack default.""" + rule = _single_rule( + """ + standard: p + default_action: deny + rules: + - id: r + hook: before_model + action: bogus + checks: [{type: regex, patterns: ["x"]}] + """ + ) + assert rule.action == Action.DENY + + +def test_severity_resolution_explicit() -> None: + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: before_model + severity: critical + checks: [{type: regex, patterns: ["x"]}] + """ + ) + assert rule.severity == Severity.CRITICAL + + +def test_severity_default_high_for_deny_action() -> None: + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: before_model + action: deny + checks: [{type: regex, patterns: ["x"]}] + """ + ) + assert rule.severity == Severity.HIGH + + +def test_severity_default_medium_for_non_deny_action() -> None: + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: before_model + action: log + checks: [{type: regex, patterns: ["x"]}] + """ + ) + assert rule.severity == Severity.MEDIUM + + +def test_unknown_severity_falls_back_to_high() -> None: + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: before_model + severity: ridiculous + checks: [{type: regex, patterns: ["x"]}] + """ + ) + assert rule.severity == Severity.HIGH + + +def test_disabled_flag_propagates() -> None: + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: before_model + enabled: false + checks: [{type: regex, patterns: ["x"]}] + """ + ) + assert rule.enabled is False + + +def test_rule_without_id_gets_index_based_id() -> None: + """When ``id:`` is missing, a positional fallback ``RULE-N`` is used.""" + idx = build_policy_index_from_yaml( + """ + standard: p + rules: + - hook: before_model + checks: [{type: regex, patterns: ["x"]}] + """ + ) + assert idx.all_rules[0].rule_id == "RULE-0" + + +def test_rule_with_zero_parsed_checks_is_skipped() -> None: + """A rule whose declared checks all fail to parse is dropped. + + Without this guard, a rule with no checks ``always matches`` in the + evaluator and would fire on every request. + """ + idx = build_policy_index_from_yaml( + """ + standard: p + rules: + - id: junk + hook: before_model + checks: + - type: totally_unknown_check_type + """ + ) + assert idx.total_rules == 0 + + +@pytest.mark.parametrize( + "hook_name,expected", + [ + ("before_agent", LifecycleHook.BEFORE_AGENT), + ("after_agent", LifecycleHook.AFTER_AGENT), + ("before_model", LifecycleHook.BEFORE_MODEL), + ("after_model", LifecycleHook.AFTER_MODEL), + ("tool_call", LifecycleHook.TOOL_CALL), + ("wrap_tool_call", LifecycleHook.TOOL_CALL), # alias + ("after_tool", LifecycleHook.AFTER_TOOL), + ], +) +def test_hook_resolution(hook_name: str, expected: LifecycleHook) -> None: + rule = _single_rule( + f""" + standard: p + rules: + - id: r + hook: {hook_name} + checks: [{{type: regex, patterns: ["x"]}}] + """ + ) + assert rule.hook == expected + + +def test_regex_check_multi_pattern_defaults_to_any_logic() -> None: + """Multiple regex patterns default to OR (any) — common case for ASI rules.""" + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: before_model + checks: + - type: regex + patterns: ["pwn", "ignore_previous"] + """ + ) + assert rule.checks[0].logic == "any" + assert len(rule.checks[0].conditions) == 2 + + +def test_regex_check_single_pattern_defaults_to_all_logic() -> None: + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: before_model + checks: + - type: regex + patterns: ["pwn"] + """ + ) + assert rule.checks[0].logic == "all" + + +def test_regex_check_explicit_logic_wins() -> None: + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: before_model + checks: + - type: regex + patterns: ["a", "b"] + logic: all + """ + ) + assert rule.checks[0].logic == "all" + + +@pytest.mark.parametrize( + "scope,expected_field", + [ + (["human"], "model_input"), + (["system"], "model_input"), + (["ai"], "model_output"), + ("ai", "model_output"), # string form + (["tool_result"], "tool_result"), + (["unknown_thing"], "model_input"), # fallback + ], +) +def test_regex_scope_maps_to_field(scope, expected_field: str) -> None: + rule = _single_rule( + f""" + standard: p + rules: + - id: r + hook: before_model + checks: + - type: regex + patterns: ["x"] + scope: {scope!r} + """ + ) + assert rule.checks[0].conditions[0].field == expected_field + + +def test_budget_check_max_per_session() -> None: + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: tool_call + checks: + - type: budget + max_tool_calls_per_session: 5 + """ + ) + cond = rule.checks[0].conditions[0] + assert cond.operator == "gt" + assert cond.field == "session_state.tool_calls" + assert cond.value == 5 + + +def test_budget_check_multiple_thresholds() -> None: + """All three budget knobs become independent conditions.""" + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: tool_call + checks: + - type: budget + max_tool_calls_per_session: 10 + max_tool_calls_per_minute: 5 + max_consecutive_tool_calls: 3 + """ + ) + assert len(rule.checks[0].conditions) == 3 + + +def test_tool_allowlist_check() -> None: + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: tool_call + checks: + - type: tool_allowlist + blocked_tools: ["delete_file", "shell"] + """ + ) + cond = rule.checks[0].conditions[0] + assert cond.operator == "in_list" + assert cond.field == "tool_name" + assert cond.value == ["delete_file", "shell"] + + +def test_tool_allowlist_empty_blocked_list_skipped() -> None: + """Empty ``blocked_tools`` means there's nothing to enforce — drop the rule.""" + idx = build_policy_index_from_yaml( + """ + standard: p + rules: + - id: r + hook: tool_call + checks: + - type: tool_allowlist + blocked_tools: [] + """ + ) + assert idx.total_rules == 0 + + +def test_parameter_validation_check() -> None: + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: tool_call + checks: + - type: parameter_validation + additional_patterns: ["rm -rf", "/etc/passwd"] + """ + ) + check = rule.checks[0] + assert len(check.conditions) == 2 + assert all(c.field == "tool_args" for c in check.conditions) + # Multi-pattern parameter_validation defaults to OR logic + assert check.logic == "any" + + +def test_rate_limit_check_session_and_minute() -> None: + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: before_model + checks: + - type: rate_limit + max_llm_calls_per_session: 20 + max_llm_calls_per_minute: 5 + """ + ) + fields = {c.field for c in rule.checks[0].conditions} + assert fields == { + "session_state.llm_calls", + "session_state.llm_calls_per_minute", + } + + +def test_field_regex_check_threads_through_conditions() -> None: + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: after_model + checks: + - type: field_regex + conditions: + - operator: regex + field: model_output + value: "(?i)password" + message: "leaked password" + """ + ) + check = rule.checks[0] + assert check.message == "leaked password" + assert check.conditions[0].operator == "regex" + + +def test_data_quality_score_both_encoding_and_entropy() -> None: + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: after_tool + checks: + - type: data_quality_score + field: tool_result + min_confidence: 0.8 + entropy_min: 2.0 + entropy_max: 6.0 + """ + ) + ops = {c.operator for c in rule.checks[0].conditions} + assert ops == {"encoding_concern", "entropy_concern"} + + +def test_data_quality_score_check_encoding_disabled() -> None: + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: after_tool + checks: + - type: data_quality_score + check_encoding: false + check_entropy: true + """ + ) + ops = [c.operator for c in rule.checks[0].conditions] + assert "encoding_concern" not in ops + assert "entropy_concern" in ops + + +def test_incident_taxonomy_with_categories() -> None: + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: after_model + checks: + - type: incident_taxonomy + field: model_output + categories: [safety_refusal, tool_failure] + """ + ) + cond = rule.checks[0].conditions[0] + assert cond.operator == "incident_concern" + assert cond.value == {"categories": ["safety_refusal", "tool_failure"]} + + +def test_incident_taxonomy_without_categories_uses_empty_dict() -> None: + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: after_model + checks: + - type: incident_taxonomy + """ + ) + cond = rule.checks[0].conditions[0] + assert cond.value == {} + + +def test_commitment_extractor_default_flags() -> None: + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: after_model + checks: + - type: commitment_extractor + """ + ) + cond = rule.checks[0].conditions[0] + assert cond.operator == "commitment_concern" + assert cond.value == {"require_amount": True, "require_deadline": False} + + +def test_commitment_extractor_custom_flags() -> None: + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: after_model + checks: + - type: commitment_extractor + require_amount: false + require_deadline: true + """ + ) + cond = rule.checks[0].conditions[0] + assert cond.value == {"require_amount": False, "require_deadline": True} + + +def test_sentiment_concern_check() -> None: + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: before_model + checks: + - type: sentiment_concern + threshold: -0.5 + """ + ) + cond = rule.checks[0].conditions[0] + assert cond.operator == "vader_concern" + assert cond.value == {"threshold": -0.5} + + +def test_guardrail_fallback_inherits_rule_flags() -> None: + """Rule-level ``mapped_to_uipath`` / ``policy_enabled`` thread into the condition.""" + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: before_model + mapped_to_uipath: true + policy_enabled: false + checks: + - type: guardrail_fallback + validator: pii_detection + """ + ) + cond = rule.checks[0].conditions[0] + assert cond.operator == "guardrail_fallback" + assert cond.value == { + "validator": "pii_detection", + "mapped_to_uipath": True, + "policy_enabled": False, + } + + +def test_guardrail_fallback_default_flags_are_unmapped_and_enabled() -> None: + """When the rule omits the flags, the fallback never fires (disabled-only contract).""" + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: before_model + checks: + - type: guardrail_fallback + validator: pii_detection + """ + ) + cond = rule.checks[0].conditions[0] + # ``guardrail_fallback`` operator fires only when mapped=True AND + # enabled=False; defaults of False / True ensure it stays silent. + assert cond.value["mapped_to_uipath"] is False + assert cond.value["policy_enabled"] is True + + +def test_explicit_conditions_win_over_check_type() -> None: + """Explicit ``conditions:`` short-circuits the per-type templating.""" + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: before_model + checks: + - type: regex # ignored, conditions wins + conditions: + - operator: contains + field: model_input + value: "secret" + message: "no secrets" + """ + ) + cond = rule.checks[0].conditions[0] + assert cond.operator == "contains" # not "regex" + assert cond.value == "secret" + assert rule.checks[0].message == "no secrets" + + +def test_explicit_conditions_negate_flag_propagates() -> None: + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: before_model + checks: + - conditions: + - operator: contains + field: model_input + value: "allowed" + negate: true + """ + ) + assert rule.checks[0].conditions[0].negate is True + + +def test_non_dict_condition_in_explicit_list_is_skipped() -> None: + """A condition entry that isn't a dict is silently dropped. + + The first dict-with-``operator`` entry is what trips the + "explicit conditions" branch in ``_build_check``; out-of-order + scalar entries appear after the leading dict. + """ + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: before_model + checks: + - conditions: + - operator: contains + field: model_input + value: "x" + - "not a dict" + """ + ) + assert len(rule.checks[0].conditions) == 1 + + +def test_unknown_check_type_skipped() -> None: + """Unknown check types are dropped without taking down sibling checks.""" + idx = build_policy_index_from_yaml( + """ + standard: p + rules: + - id: r + hook: before_model + checks: + - type: future_check_type + - type: regex + patterns: ["x"] + """ + ) + rule = idx.all_rules[0] + # Only the regex check survived. + assert len(rule.checks) == 1 + assert rule.checks[0].conditions[0].operator == "regex" + + +def test_non_dict_check_entry_skipped() -> None: + """Checks list entries that aren't dicts are silently ignored.""" + rule = _single_rule( + """ + standard: p + rules: + - id: r + hook: before_model + checks: + - "scalar instead of mapping" + - type: regex + patterns: ["x"] + """ + ) + assert len(rule.checks) == 1 diff --git a/packages/uipath/tests/cli/chat/test_bridge.py b/packages/uipath/tests/cli/chat/test_bridge.py index 2da4f31ad..2c4aedbe9 100644 --- a/packages/uipath/tests/cli/chat/test_bridge.py +++ b/packages/uipath/tests/cli/chat/test_bridge.py @@ -1,5 +1,6 @@ """Tests for SocketIOChatBridge and get_chat_bridge.""" +import asyncio import logging from datetime import datetime from typing import Any, cast @@ -9,6 +10,12 @@ from uipath._cli._chat._bridge import SocketIOChatBridge, get_chat_bridge from uipath._cli._debug._bridge import SignalRDebugBridge +from uipath.core.chat import UiPathConversationMessageEvent +from uipath.core.triggers import UiPathApiTrigger, UiPathResumeTrigger +from uipath.platform.constants import ( + HEADER_INTERNAL_ACCOUNT_ID, + HEADER_INTERNAL_TENANT_ID, +) class MockRuntimeContext: @@ -20,11 +27,13 @@ def __init__( exchange_id: str = "test-exchange-id", tenant_id: str = "test-tenant-id", org_id: str = "test-org-id", + end_exchange: bool = True, ): self.conversation_id = conversation_id self.exchange_id = exchange_id self.tenant_id = tenant_id self.org_id = org_id + self.end_exchange = end_exchange class TestSocketIOChatBridgeDebugMode: @@ -199,11 +208,58 @@ def test_get_chat_bridge_constructs_correct_headers( assert "Authorization" in bridge.headers assert "Bearer my-access-token" in bridge.headers["Authorization"] - assert "X-UiPath-Internal-TenantId" in bridge.headers - assert "X-UiPath-Internal-AccountId" in bridge.headers + assert HEADER_INTERNAL_TENANT_ID in bridge.headers + assert HEADER_INTERNAL_ACCOUNT_ID in bridge.headers assert "X-UiPath-ConversationId" in bridge.headers assert bridge.headers["X-UiPath-ConversationId"] == "conv-789" + def test_get_chat_bridge_falls_back_to_env_when_tenant_and_org_absent( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Tenant/account headers fall back to env vars when context values are None.""" + monkeypatch.setenv("UIPATH_URL", "https://cloud.uipath.com/org/tenant") + monkeypatch.setenv("UIPATH_ACCESS_TOKEN", "my-access-token") + monkeypatch.setenv("UIPATH_TENANT_ID", "env-tenant") + monkeypatch.setenv("UIPATH_ORGANIZATION_ID", "env-org") + + context = MockRuntimeContext( + tenant_id=None, # type: ignore[arg-type] + org_id=None, # type: ignore[arg-type] + conversation_id="conv-789", + ) + + bridge = cast(SocketIOChatBridge, get_chat_bridge(cast(Any, context))) + + assert bridge.headers[HEADER_INTERNAL_TENANT_ID] == "env-tenant" + assert bridge.headers[HEADER_INTERNAL_ACCOUNT_ID] == "env-org" + + def test_get_chat_bridge_includes_conversational_user_id_header_when_set( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Conversation owner id (from FpsProperties) is sent on the handshake for CAS to validate.""" + monkeypatch.setenv("UIPATH_URL", "https://cloud.uipath.com/org/tenant") + monkeypatch.setenv("UIPATH_ACCESS_TOKEN", "my-access-token") + + context = MockRuntimeContext(conversation_id="conv-789") + context.conversational_user_id = "owner-guid" # type: ignore[attr-defined] + + bridge = cast(SocketIOChatBridge, get_chat_bridge(cast(Any, context))) + + assert bridge.headers["X-UiPath-Internal-ConversationalUserId"] == "owner-guid" + + def test_get_chat_bridge_omits_conversational_user_id_header_when_absent( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """No header is sent when the runtime has no owner id (backward compatible).""" + monkeypatch.setenv("UIPATH_URL", "https://cloud.uipath.com/org/tenant") + monkeypatch.setenv("UIPATH_ACCESS_TOKEN", "my-access-token") + + context = MockRuntimeContext(conversation_id="conv-789") + + bridge = cast(SocketIOChatBridge, get_chat_bridge(cast(Any, context))) + + assert "X-UiPath-Internal-ConversationalUserId" not in bridge.headers + def test_get_chat_bridge_raises_without_uipath_url( self, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -307,6 +363,213 @@ async def test_emit_exchange_end_raises_without_client(self) -> None: assert "not connected" in str(exc_info.value).lower() + @pytest.mark.anyio + async def test_emit_message_event_sends_when_connected(self) -> None: + bridge = SocketIOChatBridge( + websocket_url="wss://test.example.com", + websocket_path="/socket.io", + conversation_id="conv-123", + exchange_id="exch-456", + headers={}, + ) + bridge._client = AsyncMock() + bridge._connected_event.set() + + await bridge.emit_message_event( + UiPathConversationMessageEvent(message_id="msg-123") + ) + + bridge._client.emit.assert_awaited_once() + + @pytest.mark.anyio + async def test_emit_exchange_error_event_sends_when_connected(self) -> None: + bridge = SocketIOChatBridge( + websocket_url="wss://test.example.com", + websocket_path="/socket.io", + conversation_id="conv-123", + exchange_id="exch-456", + headers={}, + ) + bridge._client = AsyncMock() + bridge._connected_event.set() + + await bridge.emit_exchange_error_event(ValueError("failed")) + + bridge._client.emit.assert_awaited_once() + + @pytest.mark.anyio + async def test_emit_meta_event_sends_exchange_scoped_event(self) -> None: + bridge = SocketIOChatBridge( + websocket_url="wss://test.example.com", + websocket_path="/socket.io", + conversation_id="conv-123", + exchange_id="exch-456", + headers={}, + ) + bridge._client = AsyncMock() + bridge._connected_event.set() + + await bridge.emit_meta_event( + {"workspaceFiles": [{"path": "plan.md", "attachmentKey": "key-1"}]} + ) + + bridge._client.emit.assert_awaited_once_with( + "ConversationEvent", + { + "conversationId": "conv-123", + "exchange": { + "exchangeId": "exch-456", + "metaEvent": { + "workspaceFiles": [ + {"path": "plan.md", "attachmentKey": "key-1"} + ] + }, + }, + }, + ) + + @pytest.mark.anyio + async def test_emit_meta_event_requires_client(self) -> None: + bridge = SocketIOChatBridge( + websocket_url="wss://test.example.com", + websocket_path="/socket.io", + conversation_id="conv-123", + exchange_id="exch-456", + headers={}, + ) + + with pytest.raises(RuntimeError, match="not connected"): + await bridge.emit_meta_event({}) + + @pytest.mark.anyio + async def test_emit_meta_event_requires_connected_client(self) -> None: + bridge = SocketIOChatBridge( + websocket_url="wss://test.example.com", + websocket_path="/socket.io", + conversation_id="conv-123", + exchange_id="exch-456", + headers={}, + ) + bridge._client = AsyncMock() + + with pytest.raises(RuntimeError, match="not in connected state"): + await bridge.emit_meta_event({}) + + @pytest.mark.anyio + async def test_emit_meta_event_does_not_send_in_debug_mode(self) -> None: + bridge = SocketIOChatBridge( + websocket_url="wss://test.example.com", + websocket_path="/socket.io", + conversation_id="conv-123", + exchange_id="exch-456", + headers={}, + ) + bridge._client = AsyncMock() + bridge._websocket_disabled = True + + await bridge.emit_meta_event({"workspaceFiles": []}) + + bridge._client.emit.assert_not_awaited() + + @pytest.mark.anyio + async def test_emit_meta_event_wraps_send_error(self) -> None: + bridge = SocketIOChatBridge( + websocket_url="wss://test.example.com", + websocket_path="/socket.io", + conversation_id="conv-123", + exchange_id="exch-456", + headers={}, + ) + bridge._client = AsyncMock() + bridge._client.emit.side_effect = ValueError("socket failed") + bridge._connected_event.set() + + with pytest.raises(RuntimeError, match="Failed to send conversation event"): + await bridge.emit_meta_event({}) + + +class TestSocketIOChatBridgeEndExchange: + """The bridge owns whether to honor the exchange-end event (CAS-specific).""" + + def _make_connected_bridge(self, end_exchange: bool) -> SocketIOChatBridge: + bridge = SocketIOChatBridge( + websocket_url="wss://test.example.com", + websocket_path="/socket.io", + conversation_id="conv-123", + exchange_id="exch-456", + headers={}, + end_exchange=end_exchange, + ) + bridge._websocket_disabled = False + bridge._client = AsyncMock() + bridge._connected_event.set() + return bridge + + def test_end_exchange_defaults_true(self) -> None: + bridge = SocketIOChatBridge( + websocket_url="wss://test.example.com", + websocket_path="/socket.io", + conversation_id="conv-123", + exchange_id="exch-456", + headers={}, + ) + assert bridge.end_exchange is True + + @pytest.mark.anyio + async def test_emit_exchange_end_sends_when_end_exchange_true(self) -> None: + bridge = self._make_connected_bridge(end_exchange=True) + + await bridge.emit_exchange_end_event() + + cast(AsyncMock, bridge._client).emit.assert_awaited_once() + assert ( + cast(AsyncMock, bridge._client).emit.await_args.args[0] + == "ConversationEvent" + ) + + @pytest.mark.anyio + async def test_emit_exchange_end_suppressed_when_end_exchange_false(self) -> None: + bridge = self._make_connected_bridge(end_exchange=False) + + await bridge.emit_exchange_end_event() + + cast(AsyncMock, bridge._client).emit.assert_not_awaited() + + @pytest.mark.anyio + async def test_emit_exchange_end_false_does_not_require_client(self) -> None: + """With the exchange kept open, suppression happens before the connection check.""" + bridge = SocketIOChatBridge( + websocket_url="wss://test.example.com", + websocket_path="/socket.io", + conversation_id="conv-123", + exchange_id="exch-456", + headers={}, + end_exchange=False, + ) + + # Should not raise even though _client is None. + await bridge.emit_exchange_end_event() + + def test_get_chat_bridge_propagates_end_exchange_false( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("UIPATH_URL", "https://cloud.uipath.com") + context = MockRuntimeContext(end_exchange=False) + + bridge = cast(SocketIOChatBridge, get_chat_bridge(cast(Any, context))) + + assert bridge.end_exchange is False + + def test_get_chat_bridge_defaults_end_exchange_true( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("UIPATH_URL", "https://cloud.uipath.com") + context = MockRuntimeContext() + + bridge = cast(SocketIOChatBridge, get_chat_bridge(cast(Any, context))) + + assert bridge.end_exchange is True + class TestSignalRDebugBridgeSendMethod: """Tests for SignalRDebugBridge.""" @@ -351,3 +614,589 @@ async def test_send_with_datetime_does_not_raise(self) -> None: assert parsed_data["message"] == "test message" assert isinstance(parsed_data["timestamp"], str) assert isinstance(parsed_data["nested"]["created_at"], str) + + +class TestEmitInterruptEvent: + """Tests for emit_interrupt_event — registers expected tool_call_ids.""" + + def _make_bridge(self) -> SocketIOChatBridge: + bridge = SocketIOChatBridge( + websocket_url="wss://test.example.com", + websocket_path="/socket.io", + conversation_id="conv-123", + exchange_id="exch-456", + headers={}, + ) + bridge._current_message_id = "msg-100" + return bridge + + @pytest.mark.anyio + async def test_emit_interrupt_event_does_not_emit_websocket_event(self) -> None: + """emit_interrupt_event does not emit any websocket event.""" + bridge = self._make_bridge() + + emitted_events: list[Any] = [] + + async def capture_emit(event: Any) -> None: + emitted_events.append(event) + + bridge.emit_message_event = capture_emit # type: ignore[assignment] + + trigger = UiPathResumeTrigger( + api_resume=UiPathApiTrigger( + request={ + "tool_call_id": "tc-42", + "tool_name": "my_tool", + "input": {"key": "value"}, + } + ) + ) + + await bridge.emit_interrupt_event(trigger) + + assert len(emitted_events) == 0 + + @pytest.mark.anyio + async def test_emit_interrupt_event_registers_tool_call_id(self) -> None: + """emit_interrupt_event adds the tool_call_id to the expected queue.""" + bridge = self._make_bridge() + + trigger = UiPathResumeTrigger( + api_resume=UiPathApiTrigger( + request={"tool_call_id": "tc-42", "tool_name": "my_tool"} + ) + ) + + await bridge.emit_interrupt_event(trigger) + + assert list(bridge._expected_tool_call_ids) == ["tc-42"] + + @pytest.mark.anyio + async def test_emit_interrupt_event_skips_without_tool_call_id(self) -> None: + """emit_interrupt_event does not register if tool_call_id is missing.""" + bridge = self._make_bridge() + + trigger = UiPathResumeTrigger( + api_resume=UiPathApiTrigger(request={"tool_name": "my_tool"}) + ) + + await bridge.emit_interrupt_event(trigger) + + assert len(bridge._expected_tool_call_ids) == 0 + + @pytest.mark.anyio + async def test_emit_interrupt_event_registers_multiple_in_order(self) -> None: + """Multiple emit_interrupt_event calls register IDs in FIFO order.""" + bridge = self._make_bridge() + + for tc_id in ["tc-1", "tc-2", "tc-3"]: + trigger = UiPathResumeTrigger( + api_resume=UiPathApiTrigger(request={"tool_call_id": tc_id}) + ) + await bridge.emit_interrupt_event(trigger) + + assert list(bridge._expected_tool_call_ids) == ["tc-1", "tc-2", "tc-3"] + + +class TestEmitExecutingToolCall: + """Tests for emit_executing_tool_call_event (post-confirmation executingToolCall emission).""" + + def _make_bridge(self) -> SocketIOChatBridge: + bridge = SocketIOChatBridge( + websocket_url="wss://test.example.com", + websocket_path="/socket.io", + conversation_id="conv-123", + exchange_id="exch-456", + headers={}, + ) + bridge._current_message_id = "msg-100" + return bridge + + @pytest.mark.anyio + async def test_emits_executing_tool_call_event( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Should emit executingToolCall with tool_call_id and input.""" + monkeypatch.setenv("CAS_WEBSOCKET_DISABLED", "true") + bridge = self._make_bridge() + await bridge.connect() + + emitted_events: list[Any] = [] + original_emit = bridge.emit_message_event + + async def capture_emit(event: Any) -> None: + emitted_events.append(event) + await original_emit(event) + + bridge.emit_message_event = capture_emit # type: ignore[assignment] + + await bridge.emit_executing_tool_call_event( + tool_call_id="tc-42", + tool_input={"key": "value"}, + ) + + assert len(emitted_events) == 1 + event = emitted_events[0] + assert event.message_id == "msg-100" + assert event.tool_call is not None + assert event.tool_call.tool_call_id == "tc-42" + assert event.tool_call.executing is not None + assert event.tool_call.executing.input == {"key": "value"} + + @pytest.mark.anyio + async def test_no_message_id_does_not_emit(self) -> None: + """Should not emit if no current message ID is set.""" + bridge = SocketIOChatBridge( + websocket_url="wss://test.example.com", + websocket_path="/socket.io", + conversation_id="conv-123", + exchange_id="exch-456", + headers={}, + ) + # _current_message_id is not set + + emitted_events: list[Any] = [] + + async def capture_emit(event: Any) -> None: + emitted_events.append(event) + + bridge.emit_message_event = capture_emit # type: ignore[assignment] + + await bridge.emit_executing_tool_call_event(tool_call_id="tc-42") + + assert len(emitted_events) == 0 + + @pytest.mark.anyio + async def test_none_input_emits_with_none( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Should emit with None input when no input provided.""" + monkeypatch.setenv("CAS_WEBSOCKET_DISABLED", "true") + bridge = self._make_bridge() + await bridge.connect() + + emitted_events: list[Any] = [] + original_emit = bridge.emit_message_event + + async def capture_emit(event: Any) -> None: + emitted_events.append(event) + await original_emit(event) + + bridge.emit_message_event = capture_emit # type: ignore[assignment] + + await bridge.emit_executing_tool_call_event(tool_call_id="tc-42") + + assert len(emitted_events) == 1 + assert emitted_events[0].tool_call.executing.input is None + + +class TestWaitForResumeEndToolCall: + """Tests for wait_for_resume unblocking on endToolCall events.""" + + def _make_bridge(self) -> SocketIOChatBridge: + return SocketIOChatBridge( + websocket_url="wss://test.example.com", + websocket_path="/socket.io", + conversation_id="conv-123", + exchange_id="exch-456", + headers={}, + ) + + def _make_end_event(self, tool_call_id: str, output: Any = None) -> dict[str, Any]: + return { + "conversationId": "conv-123", + "exchange": { + "exchangeId": "exch-456", + "message": { + "messageId": "msg-200", + "toolCall": { + "toolCallId": tool_call_id, + "endToolCall": { + "output": output + if output is not None + else {"result": "ok"}, + "isError": False, + }, + }, + }, + }, + } + + def _make_confirm_event(self, tool_call_id: str) -> dict[str, Any]: + return { + "conversationId": "conv-123", + "exchange": { + "exchangeId": "exch-456", + "message": { + "messageId": "msg-200", + "toolCall": { + "toolCallId": tool_call_id, + "confirmToolCall": { + "approved": True, + "input": {"edited": "data"}, + }, + }, + }, + }, + } + + async def _register(self, bridge: SocketIOChatBridge, tool_call_id: str) -> None: + """Register an expected tool_call_id via emit_interrupt_event.""" + trigger = UiPathResumeTrigger( + api_resume=UiPathApiTrigger(request={"tool_call_id": tool_call_id}) + ) + await bridge.emit_interrupt_event(trigger) + + @pytest.mark.anyio + async def test_end_tool_call_unblocks_wait_for_resume(self) -> None: + """Receiving an endToolCall event unblocks wait_for_resume and returns parsed payload.""" + bridge = self._make_bridge() + await self._register(bridge, "tc-99") + + async def simulate_end_event() -> None: + await asyncio.sleep(0.05) + await bridge._handle_conversation_event( + self._make_end_event("tc-99"), "sid-1" + ) + + task = asyncio.create_task(simulate_end_event()) + result = await bridge.wait_for_resume() + await task + + assert result["output"] == {"result": "ok"} + assert result["is_error"] is False + assert result["tool_call_id"] == "tc-99" + + @pytest.mark.anyio + async def test_confirm_tool_call_unblocks_wait_for_resume(self) -> None: + """Receiving a confirmToolCall event also unblocks wait_for_resume.""" + bridge = self._make_bridge() + await self._register(bridge, "tc-99") + + async def simulate_confirm_event() -> None: + await asyncio.sleep(0.05) + await bridge._handle_conversation_event( + self._make_confirm_event("tc-99"), "sid-1" + ) + + task = asyncio.create_task(simulate_confirm_event()) + result = await bridge.wait_for_resume() + await task + + assert result["approved"] is True + assert result["input"] == {"edited": "data"} + assert result["tool_call_id"] == "tc-99" + + @pytest.mark.anyio + async def test_early_end_tool_call_is_not_lost(self) -> None: + """An endToolCall that arrives before wait_for_resume is called must not be lost.""" + bridge = self._make_bridge() + await self._register(bridge, "tc-100") + + # Response arrives BEFORE wait_for_resume is called + await bridge._handle_conversation_event( + self._make_end_event("tc-100", output={"early": True}), "sid-1" + ) + + result = await bridge.wait_for_resume() + + assert result["output"] == {"early": True} + assert result["is_error"] is False + assert result["tool_call_id"] == "tc-100" + + @pytest.mark.anyio + async def test_concurrent_tool_calls_all_early(self) -> None: + """Multiple endToolCall responses arriving before any wait_for_resume are all preserved.""" + bridge = self._make_bridge() + + # Runtime registers 3 expected tool calls + for tc_id in ["tc-1", "tc-2", "tc-3"]: + await self._register(bridge, tc_id) + + # All 3 responses arrive before any wait_for_resume call + for tc_id in ["tc-1", "tc-2", "tc-3"]: + await bridge._handle_conversation_event( + self._make_end_event(tc_id, output={"id": tc_id}), "sid-1" + ) + + # Each wait_for_resume returns the correct result matched by tool_call_id + for tc_id in ["tc-1", "tc-2", "tc-3"]: + result = await bridge.wait_for_resume() + assert result["tool_call_id"] == tc_id + assert result["output"] == {"id": tc_id} + + # All storage is empty after consumption + assert len(bridge._tool_resume_results) == 0 + assert len(bridge._tool_resume_pending) == 0 + + @pytest.mark.anyio + async def test_concurrent_tool_calls_out_of_order(self) -> None: + """Responses arriving in reverse order are matched to the correct wait_for_resume call.""" + bridge = self._make_bridge() + + # Runtime registers in order: tc-A, tc-B, tc-C + for tc_id in ["tc-A", "tc-B", "tc-C"]: + await self._register(bridge, tc_id) + + # Responses arrive in reverse order: tc-C, tc-B, tc-A + for tc_id in ["tc-C", "tc-B", "tc-A"]: + await bridge._handle_conversation_event( + self._make_end_event(tc_id, output={"id": tc_id}), "sid-1" + ) + + # wait_for_resume consumes in registration order, each gets correct result + result_a = await bridge.wait_for_resume() + assert result_a["tool_call_id"] == "tc-A" + + result_b = await bridge.wait_for_resume() + assert result_b["tool_call_id"] == "tc-B" + + result_c = await bridge.wait_for_resume() + assert result_c["tool_call_id"] == "tc-C" + + @pytest.mark.anyio + async def test_concurrent_mixed_early_and_late(self) -> None: + """Mix of early arrivals and late arrivals are all matched correctly.""" + bridge = self._make_bridge() + + # Register 3 expected tool calls + for tc_id in ["tc-1", "tc-2", "tc-3"]: + await self._register(bridge, tc_id) + + # tc-1 arrives early (before any wait_for_resume) + await bridge._handle_conversation_event( + self._make_end_event("tc-1", output={"id": "tc-1"}), "sid-1" + ) + + # First wait_for_resume finds tc-1 already in results + result_1 = await bridge.wait_for_resume() + assert result_1["tool_call_id"] == "tc-1" + + # Second wait_for_resume blocks — tc-2 arrives while waiting + async def send_tc2() -> None: + await asyncio.sleep(0.05) + await bridge._handle_conversation_event( + self._make_end_event("tc-2", output={"id": "tc-2"}), "sid-1" + ) + + task = asyncio.create_task(send_tc2()) + result_2 = await bridge.wait_for_resume() + await task + assert result_2["tool_call_id"] == "tc-2" + + # tc-3 arrives early before third wait_for_resume + await bridge._handle_conversation_event( + self._make_end_event("tc-3", output={"id": "tc-3"}), "sid-1" + ) + result_3 = await bridge.wait_for_resume() + assert result_3["tool_call_id"] == "tc-3" + + @pytest.mark.anyio + async def test_confirm_then_end_same_tool_call(self) -> None: + """Tool with requireConversationalConfirmation: confirm and end are handled sequentially.""" + bridge = self._make_bridge() + + # Runtime registers the same tool_call_id twice (once for confirm, once for end) + await self._register(bridge, "tc-42") + await self._register(bridge, "tc-42") + + # Confirm arrives + await bridge._handle_conversation_event( + self._make_confirm_event("tc-42"), "sid-1" + ) + + # First wait_for_resume gets the confirm + result_confirm = await bridge.wait_for_resume() + assert result_confirm["approved"] is True + assert result_confirm["tool_call_id"] == "tc-42" + + # End arrives after confirm was consumed + async def send_end() -> None: + await asyncio.sleep(0.05) + await bridge._handle_conversation_event( + self._make_end_event("tc-42", output={"done": True}), "sid-1" + ) + + task = asyncio.create_task(send_end()) + result_end = await bridge.wait_for_resume() + await task + assert result_end["output"] == {"done": True} + assert result_end["tool_call_id"] == "tc-42" + + +class TestWaitForResumeEdgeCases: + """Edge case tests to ensure the resume mechanism doesn't crash.""" + + def _make_bridge(self) -> SocketIOChatBridge: + return SocketIOChatBridge( + websocket_url="wss://test.example.com", + websocket_path="/socket.io", + conversation_id="conv-123", + exchange_id="exch-456", + headers={}, + ) + + def _make_end_event(self, tool_call_id: str, output: Any = None) -> dict[str, Any]: + return { + "conversationId": "conv-123", + "exchange": { + "exchangeId": "exch-456", + "message": { + "messageId": "msg-200", + "toolCall": { + "toolCallId": tool_call_id, + "endToolCall": { + "output": output + if output is not None + else {"result": "ok"}, + "isError": False, + }, + }, + }, + }, + } + + async def _register(self, bridge: SocketIOChatBridge, tool_call_id: str) -> None: + trigger = UiPathResumeTrigger( + api_resume=UiPathApiTrigger(request={"tool_call_id": tool_call_id}) + ) + await bridge.emit_interrupt_event(trigger) + + @pytest.mark.anyio + async def test_wait_for_resume_without_registration_raises(self) -> None: + """wait_for_resume with empty deque raises RuntimeError, not IndexError.""" + bridge = self._make_bridge() + + with pytest.raises(RuntimeError, match="no tool_call_id was registered"): + await bridge.wait_for_resume() + + @pytest.mark.anyio + async def test_duplicate_response_stored_does_not_crash(self) -> None: + """Two responses for the same tool_call_id before consumption logs warning, doesn't crash.""" + bridge = self._make_bridge() + await self._register(bridge, "tc-dup") + + # First response stored + await bridge._handle_conversation_event( + self._make_end_event("tc-dup", output={"first": True}), "sid-1" + ) + # Second response overwrites (with warning), but no crash + await bridge._handle_conversation_event( + self._make_end_event("tc-dup", output={"second": True}), "sid-1" + ) + + result = await bridge.wait_for_resume() + # Second overwrote first + assert result["output"] == {"second": True} + + @pytest.mark.anyio + async def test_duplicate_response_pending_does_not_crash(self) -> None: + """Two responses while a Future is pending — first resolves it, second is stored as fallback.""" + bridge = self._make_bridge() + await self._register(bridge, "tc-dup") + + # Start waiting (creates a pending Future) + async def wait() -> dict[str, Any]: + return await bridge.wait_for_resume() + + wait_task = asyncio.create_task(wait()) + await asyncio.sleep(0.02) + + # First response resolves the Future + await bridge._handle_conversation_event( + self._make_end_event("tc-dup", output={"first": True}), "sid-1" + ) + # Second response — Future already resolved, should not crash + await bridge._handle_conversation_event( + self._make_end_event("tc-dup", output={"second": True}), "sid-1" + ) + + result = await wait_task + assert result["output"] == {"first": True} + + @pytest.mark.anyio + async def test_malformed_event_does_not_crash(self) -> None: + """Malformed conversation events are caught and don't crash the bridge.""" + bridge = self._make_bridge() + + # Completely wrong structure + await bridge._handle_conversation_event({"garbage": True}, "sid-1") + + # Missing toolCall + await bridge._handle_conversation_event( + { + "conversationId": "conv-123", + "exchange": {"exchangeId": "exch-456", "message": {"messageId": "m-1"}}, + }, + "sid-1", + ) + + # Missing endToolCall and confirmToolCall + await bridge._handle_conversation_event( + { + "conversationId": "conv-123", + "exchange": { + "exchangeId": "exch-456", + "message": { + "messageId": "m-1", + "toolCall": {"toolCallId": "tc-1"}, + }, + }, + }, + "sid-1", + ) + + # No crash — bridge is still functional + assert len(bridge._tool_resume_results) == 0 + assert len(bridge._tool_resume_pending) == 0 + + @pytest.mark.anyio + async def test_emit_interrupt_event_no_api_resume(self) -> None: + """emit_interrupt_event with no api_resume does not crash or register.""" + bridge = self._make_bridge() + + trigger = UiPathResumeTrigger() + await bridge.emit_interrupt_event(trigger) + + assert len(bridge._expected_tool_call_ids) == 0 + + @pytest.mark.anyio + async def test_emit_interrupt_event_non_dict_request(self) -> None: + """emit_interrupt_event with non-dict request does not crash or register.""" + bridge = self._make_bridge() + + trigger = UiPathResumeTrigger(api_resume=UiPathApiTrigger(request="not-a-dict")) + await bridge.emit_interrupt_event(trigger) + + assert len(bridge._expected_tool_call_ids) == 0 + + @pytest.mark.anyio + async def test_unrequested_response_stored_harmlessly(self) -> None: + """A response for an unregistered tool_call_id is stored without crashing.""" + bridge = self._make_bridge() + + # No registration, but a response arrives + await bridge._handle_conversation_event( + self._make_end_event("tc-unknown", output={"surprise": True}), "sid-1" + ) + + # Stored in results — won't be consumed but doesn't crash + assert "tc-unknown" in bridge._tool_resume_results + + @pytest.mark.anyio + async def test_storage_cleaned_up_after_consumption(self) -> None: + """After all wait_for_resume calls complete, no state is left behind.""" + bridge = self._make_bridge() + + for tc_id in ["tc-1", "tc-2", "tc-3"]: + await self._register(bridge, tc_id) + await bridge._handle_conversation_event( + self._make_end_event(tc_id), "sid-1" + ) + + for _ in range(3): + await bridge.wait_for_resume() + + assert len(bridge._tool_resume_results) == 0 + assert len(bridge._tool_resume_pending) == 0 + assert len(bridge._expected_tool_call_ids) == 0 diff --git a/packages/uipath/tests/cli/chat/test_voice_bridge.py b/packages/uipath/tests/cli/chat/test_voice_bridge.py new file mode 100644 index 000000000..c4e704922 --- /dev/null +++ b/packages/uipath/tests/cli/chat/test_voice_bridge.py @@ -0,0 +1,235 @@ +"""Tests for VoiceToolCallSession and get_voice_bridge.""" + +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from uipath._cli._chat._voice_bridge import ( + VoiceSessionEndReason, + VoiceToolCallSession, + get_voice_bridge, +) +from uipath.core.chat import ( + UiPathVoiceToolCallRequest, + UiPathVoiceToolCallResult, +) +from uipath.platform.constants import ( + HEADER_INTERNAL_ACCOUNT_ID, + HEADER_INTERNAL_TENANT_ID, +) + + +def _make_session(tool_handler: Any = None) -> VoiceToolCallSession: + session = VoiceToolCallSession( + url="wss://example/test", + socketio_path="/socket.io", + headers={}, + tool_handler=tool_handler or AsyncMock(), + ) + session._client = MagicMock() + session._client.emit = AsyncMock() + return session + + +class TestEndSession: + def test_first_writer_wins(self) -> None: + """A late DISCONNECTED must not overwrite COMPLETED.""" + session = _make_session() + session._end_session(VoiceSessionEndReason.COMPLETED) + session._end_session(VoiceSessionEndReason.DISCONNECTED) + assert session._end_reason == VoiceSessionEndReason.COMPLETED + assert session._done.is_set() + + async def test_session_ended_sets_completed(self) -> None: + session = _make_session() + await session._handle_session_ended(None) + assert session._end_reason == VoiceSessionEndReason.COMPLETED + + async def test_session_ended_preserves_payload_opaquely(self) -> None: + session = _make_session() + payload = { + "callContext": { + "type": "phone", + "id": "CA123", + "conversationId": "conv-1", + }, + "endedBy": "agent", + "callEnded": False, + "reason": "agent_completed", + "someFutureKey": {"nested": True}, + } + + await session._handle_session_ended(payload) + + assert session.end_detail == payload + assert session._end_reason == VoiceSessionEndReason.COMPLETED + returned_detail = session.end_detail + returned_detail["reason"] = "mutated" + returned_detail["callContext"]["id"] = "CA999" + payload["endedBy"] = "system" + assert session.end_detail["reason"] == "agent_completed" + assert session.end_detail["callContext"]["id"] == "CA123" + assert session.end_detail["endedBy"] == "agent" + + async def test_session_ended_preserves_output_envelope(self) -> None: + """The voice outputs envelope must reach the job runtime untouched.""" + session = _make_session() + envelope = { + "fields": {"caller_name": "Ada", "callback_requested": None}, + "status": "extracted", + "extracted": True, + } + payload = { + "callEnded": True, + "endedBy": "agent", + "reason": "agent_completed", + "endToolCalled": True, + "output": envelope, + } + + await session._handle_session_ended(payload) + + assert session.end_detail["output"] == envelope + assert session.end_detail["endToolCalled"] is True + + async def test_session_ended_non_dict_payload_is_empty_detail(self) -> None: + session = _make_session() + await session._handle_session_ended("not-a-dict") + assert session.end_detail == {} + assert session._end_reason == VoiceSessionEndReason.COMPLETED + + async def test_late_session_ended_does_not_overwrite_terminal_state(self) -> None: + session = _make_session() + await session._handle_session_ended({"endedBy": "agent", "callEnded": False}) + + await session._handle_session_ended({"endedBy": "system", "callEnded": True}) + + assert session.end_detail == {"endedBy": "agent", "callEnded": False} + assert session._end_reason == VoiceSessionEndReason.COMPLETED + + async def test_disconnect_sets_disconnected(self) -> None: + session = _make_session() + await session._handle_disconnect() + assert session._end_reason == VoiceSessionEndReason.DISCONNECTED + + +class TestHandleToolCall: + async def test_dispatches_handler_and_emits_result(self) -> None: + handler = AsyncMock( + return_value=UiPathVoiceToolCallResult(result="ok", is_error=False) + ) + session = _make_session(handler) + + await session._handle_tool_call( + {"calls": [{"callId": "c1", "toolName": "weather", "args": {"city": "SF"}}]} + ) + # Drain the spawned task. + for task in list(session._in_flight): + await task + + handler.assert_awaited_once() + assert handler.await_args is not None + call_arg = handler.await_args.args[0] + assert isinstance(call_arg, UiPathVoiceToolCallRequest) + assert call_arg.call_id == "c1" + assert call_arg.tool_name == "weather" + + session._client.emit.assert_awaited_once_with( + "voice_tool_result", + {"callId": "c1", "result": "ok", "isError": False}, + ) + + async def test_invalid_payload_is_skipped(self) -> None: + handler = AsyncMock() + session = _make_session(handler) + + await session._handle_tool_call({"calls": []}) # min_length=1 violation + + handler.assert_not_awaited() + session._client.emit.assert_not_awaited() + + async def test_noop_after_session_ended(self) -> None: + handler = AsyncMock() + session = _make_session(handler) + session._done.set() + + await session._handle_tool_call( + {"calls": [{"callId": "c1", "toolName": "x", "args": {}}]} + ) + + handler.assert_not_awaited() + assert not session._in_flight + + async def test_handler_exception_emits_error_result(self) -> None: + handler = AsyncMock(side_effect=RuntimeError("boom")) + session = _make_session(handler) + + await session._handle_tool_call( + {"calls": [{"callId": "c1", "toolName": "x", "args": {}}]} + ) + for task in list(session._in_flight): + await task + + session._client.emit.assert_awaited_once_with( + "voice_tool_result", + {"callId": "c1", "result": "boom", "isError": True}, + ) + + +class TestGetVoiceBridge: + def test_raises_when_uipath_url_missing( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("UIPATH_URL", raising=False) + monkeypatch.delenv("CAS_WEBSOCKET_HOST", raising=False) + ctx = MagicMock(conversation_id="conv-1", tenant_id="t", org_id="o") + + with pytest.raises(RuntimeError, match="UIPATH_URL"): + get_voice_bridge(ctx, AsyncMock()) + + def test_headers_fall_back_to_env_when_context_ids_are_none( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Regression: f"{None}" is truthy ("None"), so the `or` fallback was dead.""" + monkeypatch.setenv("UIPATH_URL", "https://cloud.uipath.com") + monkeypatch.setenv("UIPATH_TENANT_ID", "env-tenant") + monkeypatch.setenv("UIPATH_ORGANIZATION_ID", "env-org") + ctx = MagicMock(conversation_id="conv-1", tenant_id=None, org_id=None) + + bridge = get_voice_bridge(ctx, AsyncMock()) + + assert bridge._headers[HEADER_INTERNAL_TENANT_ID] == "env-tenant" + assert bridge._headers[HEADER_INTERNAL_ACCOUNT_ID] == "env-org" + + def test_includes_conversational_user_id_header_when_set( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Conversation owner id (from FpsProperties) is sent on the handshake for CAS to validate.""" + monkeypatch.setenv("UIPATH_URL", "https://cloud.uipath.com") + ctx = MagicMock( + conversation_id="conv-1", + tenant_id="t", + org_id="o", + conversational_user_id="owner-guid", + ) + + bridge = get_voice_bridge(ctx, AsyncMock()) + + assert bridge._headers["X-UiPath-Internal-ConversationalUserId"] == "owner-guid" + + def test_omits_conversational_user_id_header_when_none( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """No header is sent when the runtime has no owner id (backward compatible).""" + monkeypatch.setenv("UIPATH_URL", "https://cloud.uipath.com") + ctx = MagicMock( + conversation_id="conv-1", + tenant_id="t", + org_id="o", + conversational_user_id=None, + ) + + bridge = get_voice_bridge(ctx, AsyncMock()) + + assert "X-UiPath-Internal-ConversationalUserId" not in bridge._headers diff --git a/packages/uipath/tests/cli/contract/test_sdk_cli_alignment.py b/packages/uipath/tests/cli/contract/test_sdk_cli_alignment.py index 52699d42e..a690727d4 100644 --- a/packages/uipath/tests/cli/contract/test_sdk_cli_alignment.py +++ b/packages/uipath/tests/cli/contract/test_sdk_cli_alignment.py @@ -192,9 +192,13 @@ def assert_cli_sdk_alignment( # Used when SDK has optional params that CLI doesn't expose SDK_EXCLUSIONS = { "context-grounding_list": set(), - "context-grounding_retrieve": set(), + "context-grounding_retrieve": {"include_system_indexes"}, "context-grounding_create": {"source", "embeddings_enabled", "is_encrypted"}, - "context-grounding_search": {"scope", "number_of_results"}, + "context-grounding_search": { + "scope", + "number_of_results", + "include_system_indexes", + }, "context-grounding_ingest": set(), "context-grounding_delete": set(), "context-grounding_deep-rag_start": set(), diff --git a/packages/uipath/tests/cli/eval/mocks/test_input_mocker.py b/packages/uipath/tests/cli/eval/mocks/test_input_mocker.py index 72b3765df..7abdf78b1 100644 --- a/packages/uipath/tests/cli/eval/mocks/test_input_mocker.py +++ b/packages/uipath/tests/cli/eval/mocks/test_input_mocker.py @@ -68,6 +68,13 @@ async def test_generate_llm_input_with_model_settings( json={}, ) + # Chat completions consults discovery to learn which parameters the model accepts. + httpx_mock.add_response( + url="https://example.com/llm/api/discovery", + status_code=200, + json=[{"modelName": "gpt-4o-mini-2024-07-18", "modelDetails": {}}], + ) + httpx_mock.add_response( url="https://example.com/llm/api/chat/completions" "?api-version=2024-08-01-preview", @@ -112,3 +119,10 @@ async def test_generate_llm_input_with_model_settings( assert len(chat_completion_requests) == 1, ( "Expected exactly one chat completion request" ) + + # OpenAI returns content via response_format; no tool-call fallback needed. + import json + + body = json.loads(chat_completion_requests[0].content.decode("utf-8")) + assert "response_format" in body + assert "tools" not in body diff --git a/packages/uipath/tests/cli/eval/mocks/test_input_mocker_span.py b/packages/uipath/tests/cli/eval/mocks/test_input_mocker_span.py index 19a432fef..d02c5d242 100644 --- a/packages/uipath/tests/cli/eval/mocks/test_input_mocker_span.py +++ b/packages/uipath/tests/cli/eval/mocks/test_input_mocker_span.py @@ -212,6 +212,14 @@ async def test_simulate_input_span_on_error(httpx_mock: HTTPXMock, monkeypatch): }, }, ) + # The prose content above triggers the tool-call fallback; an empty + # response there fails the fallback too, producing the error span. + httpx_mock.add_response( + url="https://example.com/llm/api/chat/completions" + "?api-version=2024-08-01-preview", + status_code=200, + json={}, + ) mocking_strategy = InputMockingStrategy( prompt="Generate input", diff --git a/packages/uipath/tests/cli/eval/mocks/test_mockable_arg_collision.py b/packages/uipath/tests/cli/eval/mocks/test_mockable_arg_collision.py new file mode 100644 index 000000000..838e9d835 --- /dev/null +++ b/packages/uipath/tests/cli/eval/mocks/test_mockable_arg_collision.py @@ -0,0 +1,107 @@ +"""Regression tests: @mockable must not collide with user args named `func`/`params`.""" + +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest + +from uipath.eval.mocks import mockable +from uipath.eval.mocks._mock_runtime import ( + clear_execution_context, + set_execution_context, +) +from uipath.eval.mocks._types import MockingContext +from uipath.eval.models.evaluation_set import EvaluationItem + +_mock_span_collector = MagicMock() + + +def _build_evaluation( + function_name: str, kwargs: dict[str, Any], value: Any +) -> EvaluationItem: + evaluation_item: dict[str, Any] = { + "id": "evaluation-id", + "name": "Test evaluation", + "inputs": {}, + "evaluationCriterias": {"ExactMatchEvaluator": None}, + "mockingStrategy": { + "type": "mockito", + "behaviors": [ + { + "function": function_name, + "arguments": {"args": [], "kwargs": kwargs}, + "then": [{"type": "return", "value": value}], + } + ], + }, + } + return EvaluationItem(**evaluation_item) + + +class TestMockableArgCollision: + """Ensure `@mockable` works when the wrapped function has args named `func` or `params`.""" + + def test_sync_function_with_func_and_params_args(self): + """A sync mockable function that takes `func` and `params` kwargs should not raise.""" + + @mockable() + def test_function(func: str, params: dict[str, Any]) -> str: + raise NotImplementedError() + + evaluation = _build_evaluation( + "test_function", + kwargs={"func": "some_func", "params": {"k": "v"}}, + value="mocked_result", + ) + + set_execution_context( + MockingContext( + strategy=evaluation.mocking_strategy, + name=evaluation.name, + inputs=evaluation.inputs, + ), + _mock_span_collector, + "test-execution-id", + ) + + try: + with patch("uipath.eval.mocks.mockable.UiPathSpanUtils"): + with patch("uipath.eval.mocks.mockable.trace"): + result = test_function(func="some_func", params={"k": "v"}) + + assert result == "mocked_result" + finally: + clear_execution_context() + + @pytest.mark.asyncio + async def test_async_function_with_func_and_params_args(self): + """An async mockable function that takes `func` and `params` kwargs should not raise.""" + + @mockable() + async def test_function(func: str, params: dict[str, Any]) -> str: + raise NotImplementedError() + + evaluation = _build_evaluation( + "test_function", + kwargs={"func": "some_func", "params": {"k": "v"}}, + value="mocked_result", + ) + + set_execution_context( + MockingContext( + strategy=evaluation.mocking_strategy, + name=evaluation.name, + inputs=evaluation.inputs, + ), + _mock_span_collector, + "test-execution-id", + ) + + try: + with patch("uipath.eval.mocks.mockable.UiPathSpanUtils"): + with patch("uipath.eval.mocks.mockable.trace"): + result = await test_function(func="some_func", params={"k": "v"}) + + assert result == "mocked_result" + finally: + clear_execution_context() diff --git a/packages/uipath/tests/cli/eval/mocks/test_mocker_arg_serialization.py b/packages/uipath/tests/cli/eval/mocks/test_mocker_arg_serialization.py new file mode 100644 index 000000000..a8752c276 --- /dev/null +++ b/packages/uipath/tests/cli/eval/mocks/test_mocker_arg_serialization.py @@ -0,0 +1,251 @@ +"""Tool arguments must survive simulation regardless of their Python type. + +A simulated tool call is serialized twice: once by ``LLMMocker`` to build the +simulation prompt, and once by ``SimulateComponentService`` to POST the payload. +Tool ``args_schema`` models may type a field as any Python type (``uuid.UUID``, +``datetime``, ``Enum``, ...), and LangChain hands the tool the *validated* +value, so both paths receive objects the stdlib JSON encoder rejects. +""" + +import json +import uuid +from datetime import datetime, timezone +from enum import Enum +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from _pytest.monkeypatch import MonkeyPatch +from pytest_httpx import HTTPXMock + +from uipath.eval.mocks._mock_runtime import ( + clear_execution_context, + set_execution_context, +) +from uipath.eval.mocks._simulate_component_service import ( + _create_simulate_component_service, +) +from uipath.eval.mocks._types import ( + ComponentSimulationConfig, + LLMMockingStrategy, + MockingContext, + SimulationStrategy, + ToolSimulation, +) +from uipath.eval.mocks.mockable import mockable + +_mock_span_collector = MagicMock() + +_ATTACHMENT_ID = uuid.UUID("9b702dc7-4988-4fc0-ba81-08deeaade3da") + + +class _Flavor(str, Enum): + PDF = "pdf" + + +class _Rank(Enum): + """A plain Enum: not a str subclass, so json.dumps cannot encode it.""" + + HIGH = 1 + + +def _llm_context(tool_name: str) -> MockingContext: + return MockingContext( + strategy=LLMMockingStrategy( + prompt="simulate it", + tools_to_simulate=[ToolSimulation(name=tool_name)], + ), + name="test-run", + inputs={}, + ) + + +class TestLLMMockerArgSerialization: + """LLMMocker builds its prompt with json.dumps over the raw invocation.""" + + def teardown_method(self): + clear_execution_context() + + @pytest.mark.asyncio + async def test_uuid_argument_does_not_break_prompt_generation(self): + captured: dict[str, Any] = {} + + async def _fake_generate(llm, messages, **kwargs): + captured["prompt"] = messages[0]["content"] + return {"ok": True} + + @mockable() + async def extraction_tool(**kwargs: Any) -> dict[str, Any]: + raise NotImplementedError("must be simulated, never executed") + + set_execution_context( + _llm_context("extraction_tool"), _mock_span_collector, "exec-uuid" + ) + with ( + patch("uipath.eval.mocks._llm_mocker.UiPath", MagicMock()), + patch("uipath.eval.mocks._llm_mocker.UiPathLlmChatService", MagicMock()), + patch( + "uipath.eval.mocks._llm_mocker.generate_structured_output", + _fake_generate, + ), + ): + result = await extraction_tool( + id=_ATTACHMENT_ID, + full_name="PO_234.pdf", + mime_type="application/pdf", + ) + + assert result == {"ok": True} + # The UUID must reach the prompt in its string form. + assert str(_ATTACHMENT_ID) in captured["prompt"] + + @pytest.mark.asyncio + async def test_other_non_json_native_arguments_are_serialized(self): + captured: dict[str, Any] = {} + + async def _fake_generate(llm, messages, **kwargs): + captured["prompt"] = messages[0]["content"] + return {"ok": True} + + @mockable() + async def typed_tool(**kwargs: Any) -> dict[str, Any]: + raise NotImplementedError() + + set_execution_context( + _llm_context("typed_tool"), _mock_span_collector, "exec-typed" + ) + with ( + patch("uipath.eval.mocks._llm_mocker.UiPath", MagicMock()), + patch("uipath.eval.mocks._llm_mocker.UiPathLlmChatService", MagicMock()), + patch( + "uipath.eval.mocks._llm_mocker.generate_structured_output", + _fake_generate, + ), + ): + await typed_tool( + when=datetime(2026, 7, 27, 12, 39, 59, tzinfo=timezone.utc), + flavor=_Flavor.PDF, + rank=_Rank.HIGH, + tags={"alpha", "beta"}, + ) + + prompt = captured["prompt"] + assert "2026-07-27T12:39:59" in prompt + assert "pdf" in prompt + # A plain (non-str) Enum must be reduced to its value. + assert '"rank": 1' in prompt + # Sets become lists; assert on members so the check stays order-independent. + assert '"alpha"' in prompt + assert '"beta"' in prompt + + +class TestSimulateComponentPayloadSerialization: + """The simulate-component payload is encoded by httpx with a bare json.dumps.""" + + def teardown_method(self): + clear_execution_context() + + @pytest.mark.asyncio + @pytest.mark.httpx_mock(assert_all_responses_were_requested=False) + async def test_uuid_in_payload_is_sent_as_string( + self, httpx_mock: HTTPXMock, monkeypatch: MonkeyPatch + ): + monkeypatch.setenv("UIPATH_URL", "https://example.com/myorg/mytenant") + monkeypatch.setenv("UIPATH_ACCESS_TOKEN", "token") + httpx_mock.add_response( + url="https://example.com/myorg/mytenant/agentsruntime_/api/execution/simulations/simulate-component", + method="POST", + json={"status": 1, "simulatedOutput": "result"}, + ) + + service = _create_simulate_component_service() + result = await service.simulate( + { + "componentId": "extraction_tool", + "input": { + "args": [], + "kwargs": { + "id": _ATTACHMENT_ID, + "when": datetime(2026, 7, 27, 12, 39, 59, tzinfo=timezone.utc), + "rank": _Rank.HIGH, + "tags": {"alpha", "beta"}, + }, + }, + } + ) + + assert result == {"status": 1, "simulatedOutput": "result"} + sent_kwargs = json.loads(httpx_mock.get_requests()[-1].read())["input"][ + "kwargs" + ] + assert sent_kwargs["id"] == str(_ATTACHMENT_ID) + assert sent_kwargs["when"].startswith("2026-07-27T12:39:59") + assert sent_kwargs["rank"] == 1 + assert sorted(sent_kwargs["tags"]) == ["alpha", "beta"] + + @pytest.mark.asyncio + async def test_mocker_end_to_end_with_uuid_argument(self): + captured: list[dict[str, Any]] = [] + + async def _capture(payload, **kwargs): + captured.append(payload) + return {"status": 1, "simulatedOutput": "ok"} + + svc_mock = MagicMock() + svc_mock.simulate = _capture + + @mockable() + async def extraction_tool(**kwargs: Any) -> str: + raise NotImplementedError() + + context = MockingContext( + strategy=None, + name="test-run", + inputs={}, + workload_id="wl-1", + components=[ + ComponentSimulationConfig( + component_id="extraction_tool", + simulation_strategy=SimulationStrategy.LLM, + ) + ], + ) + set_execution_context(context, _mock_span_collector, "exec-e2e") + with patch( + "uipath.eval.mocks._simulate_component_mocker._create_simulate_component_service", + return_value=svc_mock, + ): + result = await extraction_tool(id=_ATTACHMENT_ID) + + assert result == "ok" + assert captured[0]["input"]["kwargs"]["id"] == _ATTACHMENT_ID + + +class TestSimulateComponentServiceUnchangedBehaviour: + """Normalization must not alter payloads that already serialized cleanly.""" + + @pytest.mark.asyncio + @pytest.mark.httpx_mock(assert_all_responses_were_requested=False) + async def test_json_native_payload_is_unchanged( + self, httpx_mock: HTTPXMock, monkeypatch: MonkeyPatch + ): + monkeypatch.setenv("UIPATH_URL", "https://example.com/myorg/mytenant") + monkeypatch.setenv("UIPATH_ACCESS_TOKEN", "token") + httpx_mock.add_response( + url="https://example.com/myorg/mytenant/agentsruntime_/api/execution/simulations/simulate-component", + method="POST", + json={"status": 1, "simulatedOutput": "result"}, + ) + + payload: dict[str, Any] = { + "componentId": "my_tool", + "componentType": "tool", + "input": {"args": [1, "two", True, None], "kwargs": {"nested": {"a": 1.5}}}, + "behaviors": None, + "simulationStrategy": 0, + } + service = _create_simulate_component_service() + await service.simulate(payload) + + sent = json.loads(httpx_mock.get_requests()[-1].read()) + assert sent == payload diff --git a/packages/uipath/tests/cli/eval/mocks/test_mocks.py b/packages/uipath/tests/cli/eval/mocks/test_mocks.py index bdbdd3dc2..e59b07d2f 100644 --- a/packages/uipath/tests/cli/eval/mocks/test_mocks.py +++ b/packages/uipath/tests/cli/eval/mocks/test_mocks.py @@ -610,12 +610,14 @@ def foofoo(*args, **kwargs): with pytest.raises(NotImplementedError): assert foofoo() - httpx_mock.add_response( - url="https://example.com/llm/api/chat/completions" - "?api-version=2024-08-01-preview", - status_code=200, - json={}, - ) + # Two empty responses: the response_format attempt and the tool-call fallback. + for _ in range(2): + httpx_mock.add_response( + url="https://example.com/llm/api/chat/completions" + "?api-version=2024-08-01-preview", + status_code=200, + json={}, + ) with pytest.raises(UiPathMockResponseGenerationError): assert foo() @@ -720,12 +722,14 @@ async def foofoo(*args, **kwargs): with pytest.raises(NotImplementedError): assert await foofoo() - httpx_mock.add_response( - url="https://example.com/llm/api/chat/completions" - "?api-version=2024-08-01-preview", - status_code=200, - json={}, - ) + # Two empty responses: the response_format attempt and the tool-call fallback. + for _ in range(2): + httpx_mock.add_response( + url="https://example.com/llm/api/chat/completions" + "?api-version=2024-08-01-preview", + status_code=200, + json={}, + ) with pytest.raises(UiPathMockResponseGenerationError): assert await foo() @@ -929,3 +933,230 @@ async def foo(*args, **kwargs) -> dict[str, Any]: }, }, } + + +@pytest.mark.asyncio +@pytest.mark.httpx_mock(assert_all_responses_were_requested=False) +async def test_llm_mockable_uses_tool_call_directly_for_non_openai( + httpx_mock: HTTPXMock, monkeypatch: MonkeyPatch +): + """Tool simulation works for non-OpenAI providers (AE-1646). + + Non-OpenAI providers don't honor ``response_format`` on the normalized + gateway (Claude answers with prose, Gemini with empty content), so their + strategies go straight to a forced tool call — a single request. + """ + monkeypatch.setenv("UIPATH_URL", "https://example.com") + monkeypatch.setenv("UIPATH_ACCESS_TOKEN", "1234567890") + monkeypatch.setattr(CacheManager, "get", lambda *args, **kwargs: None) + monkeypatch.setattr(CacheManager, "set", lambda *args, **kwargs: None) + + @mockable() + async def foo(*args, **kwargs) -> str: + raise NotImplementedError() + + evaluation_item: dict[str, Any] = { + "id": "evaluation-id", + "name": "Mock foo", + "inputs": {}, + "evaluationCriterias": { + "ExactMatchEvaluator": None, + }, + "mockingStrategy": { + "type": "llm", + "prompt": "response is 'bar1'", + "toolsToSimulate": [{"name": "foo"}], + "model": {"model": "anthropic.claude-sonnet-4-5-20250929-v1:0"}, + }, + } + evaluation = EvaluationItem(**evaluation_item) + assert isinstance(evaluation.mocking_strategy, LLMMockingStrategy) + httpx_mock.add_response( + url="https://example.com/agenthub_/llm/api/capabilities", + status_code=200, + json={}, + ) + httpx_mock.add_response( + url="https://example.com/orchestrator_/llm/api/capabilities", + status_code=200, + json={}, + ) + + def _completion(message: dict[str, Any]) -> dict[str, Any]: + return { + "id": "response-id", + "object": "", + "created": 0, + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "choices": [{"index": 0, "message": message, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + + # Claude goes straight to function calling: one request, one response. + httpx_mock.add_response( + url="https://example.com/llm/api/chat/completions" + "?api-version=2024-08-01-preview", + status_code=200, + json=_completion( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "name": "submit_tool_response", + "arguments": {"response": "bar1"}, + } + ], + } + ), + ) + + set_execution_context( + MockingContext( + strategy=evaluation.mocking_strategy, + name=evaluation.name, + inputs=evaluation.inputs, + ), + _mock_span_collector, + "test-execution-id", + ) + + assert await foo() == "bar1" + + requests = [ + r for r in httpx_mock.get_requests() if "chat/completions" in str(r.url) + ] + assert len(requests) == 1 + body = json.loads(requests[0].content.decode("utf-8")) + # Non-OpenAI providers use a forced tool call directly — no response_format. + assert body["tool_choice"] == {"type": "required"} + assert body["tools"][0]["name"] == "submit_tool_response" + assert "response_format" not in body + + +class TestUiPathMockRuntime: + """Tests for UiPathMockRuntime execute/stream/get_schema paths.""" + + def _make_context(self) -> MockingContext: + return MockingContext( + strategy=LLMMockingStrategy( + prompt="test", + tools_to_simulate=[ToolSimulation(name="my_tool")], + ), + name="test", + inputs={}, + ) + + async def test_execute_with_mocking_context_sets_and_clears(self): + from unittest.mock import AsyncMock, patch + + from uipath.eval.mocks._mock_runtime import UiPathMockRuntime + + delegate = MagicMock() + mock_result = MagicMock() + delegate.execute = AsyncMock(return_value=mock_result) + + runtime = UiPathMockRuntime( + delegate=delegate, + mocking_context=self._make_context(), + ) + + with ( + patch("uipath.eval.mocks._mock_runtime.set_execution_context") as mock_set, + patch( + "uipath.eval.mocks._mock_runtime.clear_execution_context" + ) as mock_clear, + ): + result = await runtime.execute({"key": "value"}) + + assert result is mock_result + mock_set.assert_called_once() + mock_clear.assert_called_once() + + async def test_stream_with_mocking_context_sets_and_clears(self): + from unittest.mock import patch + + from uipath.eval.mocks._mock_runtime import UiPathMockRuntime + + sentinel = object() + + async def _gen(*args, **kwargs): + yield sentinel + + delegate = MagicMock() + delegate.stream = _gen + + runtime = UiPathMockRuntime( + delegate=delegate, + mocking_context=self._make_context(), + ) + + with ( + patch("uipath.eval.mocks._mock_runtime.set_execution_context") as mock_set, + patch( + "uipath.eval.mocks._mock_runtime.clear_execution_context" + ) as mock_clear, + ): + events = [e async for e in runtime.stream({})] + + assert events == [sentinel] + mock_set.assert_called_once() + mock_clear.assert_called_once() + + async def test_stream_without_mocking_context_passes_through(self): + from unittest.mock import patch + + from uipath.eval.mocks._mock_runtime import UiPathMockRuntime + + sentinel = object() + + async def _gen(*args, **kwargs): + yield sentinel + + delegate = MagicMock() + delegate.stream = _gen + + runtime = UiPathMockRuntime(delegate=delegate, mocking_context=None) + with patch( + "uipath.eval.mocks._mock_runtime.load_simulation_config", return_value=None + ): + runtime._mocking_context = None + events = [e async for e in runtime.stream({})] + + assert events == [sentinel] + + async def test_get_schema_delegates(self): + from unittest.mock import AsyncMock, patch + + from uipath.eval.mocks._mock_runtime import UiPathMockRuntime + + schema = MagicMock() + delegate = MagicMock() + delegate.get_schema = AsyncMock(return_value=schema) + + runtime = UiPathMockRuntime(delegate=delegate, mocking_context=None) + with patch( + "uipath.eval.mocks._mock_runtime.load_simulation_config", return_value=None + ): + result = await runtime.get_schema() + + assert result is schema + + def test_set_execution_context_handles_mocker_creation_failure(self): + from unittest.mock import patch + + from uipath.eval._execution_context import ExecutionSpanCollector + from uipath.eval.mocks._mock_context import mocker_context + from uipath.eval.mocks._mock_runtime import set_execution_context + + context = self._make_context() + with patch( + "uipath.eval.mocks._mock_runtime.MockerFactory.create", + side_effect=RuntimeError("boom"), + ): + set_execution_context(context, ExecutionSpanCollector(), "test-id") + + # mocking_context is set, but mocker_context must be None on failure + assert mocker_context.get() is None + clear_execution_context() diff --git a/packages/uipath/tests/cli/eval/mocks/test_simulate_component.py b/packages/uipath/tests/cli/eval/mocks/test_simulate_component.py new file mode 100644 index 000000000..f4131cf48 --- /dev/null +++ b/packages/uipath/tests/cli/eval/mocks/test_simulate_component.py @@ -0,0 +1,442 @@ +"""Tests for SimulateComponentMocker and SimulateComponentService.""" + +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from _pytest.monkeypatch import MonkeyPatch +from pytest_httpx import HTTPXMock + +from uipath.eval.mocks._mock_context import is_tool_simulated +from uipath.eval.mocks._mock_runtime import ( + clear_execution_context, + set_execution_context, +) +from uipath.eval.mocks._mocker import ( + UiPathMockResponseGenerationError, +) +from uipath.eval.mocks._simulate_component_mocker import SimulateComponentMocker +from uipath.eval.mocks._simulate_component_service import ( + SimulateComponentService, + _create_simulate_component_service, +) +from uipath.eval.mocks._types import ( + ComponentSimulationConfig, + MockingContext, + SimulationStrategy, + UnknownMockingStrategy, +) +from uipath.eval.mocks.mockable import mockable + +_mock_span_collector = MagicMock() + +BASE_URL = "https://example.com" +_SIMULATE_PATH = ( + "uipath.eval.mocks._simulate_component_mocker._create_simulate_component_service" +) + + +def _make_context( + component_id: str = "my_tool", + strategy: SimulationStrategy = SimulationStrategy.LLM, + instruction: str = "simulate it", + workload_id: str = "wl-123", +) -> MockingContext: + return MockingContext( + strategy=None, + name="test-run", + inputs={"q": "hello"}, + workload_id=workload_id, + components=[ + ComponentSimulationConfig( + component_id=component_id, + component_type="tool", + simulation_strategy=strategy, + simulation_instruction=instruction, + ) + ], + ) + + +def _make_service_mock(result: dict[str, Any]) -> MagicMock: + svc = MagicMock() + svc.simulate = AsyncMock(return_value=result) + return svc + + +# --------------------------------------------------------------------------- +# is_tool_simulated with components format +# --------------------------------------------------------------------------- + + +class TestIsToolSimulatedWithComponents: + def setup_method(self): + clear_execution_context() + + def teardown_method(self): + clear_execution_context() + + def test_returns_true_for_listed_component(self): + set_execution_context(_make_context("search_tool"), _mock_span_collector, "x") + assert is_tool_simulated("search_tool") is True + + def test_returns_false_for_unlisted_component(self): + set_execution_context(_make_context("search_tool"), _mock_span_collector, "x") + assert is_tool_simulated("other_tool") is False + + def test_underscore_space_normalisation(self): + ctx = MockingContext( + strategy=None, + name="run", + inputs={}, + components=[ + ComponentSimulationConfig( + component_id="web search", + simulation_strategy=SimulationStrategy.LLM, + ) + ], + ) + set_execution_context(ctx, _mock_span_collector, "x") + assert is_tool_simulated("web_search") is True + + def test_returns_false_when_components_list_is_empty(self): + ctx = MockingContext(strategy=None, name="run", inputs={}, components=[]) + set_execution_context(ctx, _mock_span_collector, "x") + # components is set but empty — MockerFactory won't create a mocker (components is not None) + # is_tool_simulated: ctx.components is not None → iterates empty list → False + assert is_tool_simulated("any_tool") is False + + +# --------------------------------------------------------------------------- +# SimulateComponentMocker._find_component +# --------------------------------------------------------------------------- + + +class TestFindComponent: + def test_finds_by_exact_id(self): + mocker = SimulateComponentMocker(_make_context("my_tool")) + assert mocker._find_component("my_tool") is not None + + def test_finds_by_underscore_to_space_normalisation(self): + ctx = MockingContext( + strategy=None, + name="run", + inputs={}, + components=[ + ComponentSimulationConfig( + component_id="web search", + simulation_strategy=SimulationStrategy.LLM, + ) + ], + ) + mocker = SimulateComponentMocker(ctx) + assert mocker._find_component("web_search") is not None + + def test_returns_none_for_unknown_tool(self): + mocker = SimulateComponentMocker(_make_context("my_tool")) + assert mocker._find_component("unknown") is None + + +# --------------------------------------------------------------------------- +# SimulateComponentMocker.response — success path +# --------------------------------------------------------------------------- + + +class TestSimulateComponentMockerResponse: + @pytest.mark.asyncio + async def test_returns_simulated_output_on_status_1(self): + ctx = _make_context("my_tool") + svc_mock = _make_service_mock({"status": 1, "simulatedOutput": "hello"}) + + @mockable() + async def my_tool() -> str: + raise NotImplementedError() + + set_execution_context(ctx, _mock_span_collector, "exec-1") + with patch(_SIMULATE_PATH, return_value=svc_mock): + result = await my_tool() + + assert result == "hello" + clear_execution_context() + + @pytest.mark.asyncio + async def test_raises_generation_error_on_non_1_status(self): + ctx = _make_context("my_tool") + svc_mock = _make_service_mock( + {"status": 2, "error": {"message": "LLM timeout"}} + ) + + @mockable() + async def my_tool() -> str: + raise NotImplementedError() + + set_execution_context(ctx, _mock_span_collector, "exec-2") + with patch(_SIMULATE_PATH, return_value=svc_mock): + with pytest.raises(UiPathMockResponseGenerationError, match="LLM timeout"): + await my_tool() + + clear_execution_context() + + @pytest.mark.asyncio + async def test_raises_generic_error_when_error_message_missing(self): + ctx = _make_context("my_tool") + svc_mock = _make_service_mock({"status": 0, "error": {}}) + + @mockable() + async def my_tool() -> str: + raise NotImplementedError() + + set_execution_context(ctx, _mock_span_collector, "exec-3") + with patch(_SIMULATE_PATH, return_value=svc_mock): + with pytest.raises( + UiPathMockResponseGenerationError, match="Simulation failed" + ): + await my_tool() + + clear_execution_context() + + @pytest.mark.asyncio + async def test_raises_generation_error_when_api_throws(self): + ctx = _make_context("my_tool") + svc_mock = MagicMock() + svc_mock.simulate = AsyncMock(side_effect=RuntimeError("network error")) + + @mockable() + async def my_tool() -> str: + raise NotImplementedError() + + set_execution_context(ctx, _mock_span_collector, "exec-4") + with patch(_SIMULATE_PATH, return_value=svc_mock): + with pytest.raises( + UiPathMockResponseGenerationError, + match="simulate-component API call failed", + ): + await my_tool() + + clear_execution_context() + + @pytest.mark.asyncio + async def test_raises_no_mock_found_for_unconfigured_tool(self): + ctx = _make_context("my_tool") + + @mockable() + async def other_tool() -> str: + raise NotImplementedError() + + set_execution_context(ctx, _mock_span_collector, "exec-5") + # other_tool is not in components → falls through to real function + with pytest.raises(NotImplementedError): + await other_tool() + + clear_execution_context() + + +# --------------------------------------------------------------------------- +# Payload construction +# --------------------------------------------------------------------------- + + +class TestPayloadConstruction: + @pytest.mark.asyncio + async def test_payload_fields_sent_to_service(self): + ctx = _make_context("my_tool", instruction="Do something", workload_id="wl-99") + captured: list[dict[str, Any]] = [] + + async def _capture(payload, **kwargs): + captured.append(payload) + return {"status": 1, "simulatedOutput": "ok"} + + svc_mock = MagicMock() + svc_mock.simulate = _capture + + @mockable() + async def my_tool(x: int) -> str: + raise NotImplementedError() + + set_execution_context(ctx, _mock_span_collector, "exec-6") + with patch(_SIMULATE_PATH, return_value=svc_mock): + await my_tool(x=42) + + assert len(captured) == 1 + p = captured[0] + assert p["workloadId"] == "wl-99" + assert p["componentId"] == "my_tool" + assert p["componentType"] == "tool" + assert p["simulationInstruction"] == "Do something" + assert p["simulationStrategy"] == int(SimulationStrategy.LLM) + assert p["workloadInfo"] == {"name": "test-run", "userInput": {"q": "hello"}} + + clear_execution_context() + + @pytest.mark.asyncio + async def test_sync_mockable_also_works(self): + ctx = _make_context("sync_tool") + svc_mock = _make_service_mock({"status": 1, "simulatedOutput": 42}) + + @mockable() + def sync_tool() -> int: + raise NotImplementedError() + + set_execution_context(ctx, _mock_span_collector, "exec-7") + with patch(_SIMULATE_PATH, return_value=svc_mock): + result = sync_tool() + + assert result == 42 + clear_execution_context() + + @pytest.mark.asyncio + async def test_payload_uses_configured_component_id_not_invoked_name(self): + """componentId in payload must be the configured ID, not the normalised call name.""" + ctx = MockingContext( + strategy=None, + name="run", + inputs={}, + components=[ + ComponentSimulationConfig( + component_id="web search", + simulation_strategy=SimulationStrategy.LLM, + ) + ], + ) + captured: list[dict[str, Any]] = [] + + async def _capture(payload, **kwargs): + captured.append(payload) + return {"status": 1, "simulatedOutput": "ok"} + + svc_mock = MagicMock() + svc_mock.simulate = _capture + + @mockable() + async def web_search() -> str: + raise NotImplementedError() + + set_execution_context(ctx, _mock_span_collector, "exec-8") + with patch(_SIMULATE_PATH, return_value=svc_mock): + await web_search() + + assert captured[0]["componentId"] == "web search" + clear_execution_context() + + +# --------------------------------------------------------------------------- +# _build_execution_history — uncovered branch (no context vars set) +# --------------------------------------------------------------------------- + + +class TestBuildExecutionHistory: + def test_returns_none_when_context_vars_not_set(self): + clear_execution_context() + mocker = SimulateComponentMocker(_make_context()) + assert mocker._build_execution_history() is None + + def test_returns_none_when_spans_empty(self): + from uipath.eval._execution_context import ( + execution_id_context, + span_collector_context, + ) + + span_collector = MagicMock() + span_collector.get_spans = MagicMock(return_value=[]) + span_collector_context.set(span_collector) + execution_id_context.set("exec-id") + + mocker = SimulateComponentMocker(_make_context()) + assert mocker._build_execution_history() is None + + clear_execution_context() + + +# --------------------------------------------------------------------------- +# MockerFactory — unknown strategy raises ValueError +# --------------------------------------------------------------------------- + + +class TestMockerFactory: + def test_raises_for_unknown_strategy(self): + from uipath.eval.mocks._mocker_factory import MockerFactory + + ctx = MockingContext( + strategy=UnknownMockingStrategy(type="future_strategy"), + name="test", + inputs={}, + components=None, + ) + with pytest.raises(ValueError, match="Unknown mocking strategy"): + MockerFactory.create(ctx) + + def test_raises_for_none_strategy_and_no_components(self): + from uipath.eval.mocks._mocker_factory import MockerFactory + + ctx = MockingContext(strategy=None, name="test", inputs={}, components=None) + with pytest.raises(ValueError, match="Unknown mocking strategy"): + MockerFactory.create(ctx) + + +# --------------------------------------------------------------------------- +# is_tool_simulated — unknown strategy falls through to False +# --------------------------------------------------------------------------- + + +class TestIsToolSimulatedUnknownStrategy: + def setup_method(self): + clear_execution_context() + + def teardown_method(self): + clear_execution_context() + + def test_returns_false_for_unknown_strategy(self): + + ctx = MockingContext( + strategy=UnknownMockingStrategy(type="future_strategy"), + name="test", + inputs={}, + components=None, + ) + set_execution_context(ctx, _mock_span_collector, "x") + assert is_tool_simulated("any_tool") is False + + +# --------------------------------------------------------------------------- +# SimulateComponentService — actual HTTP call +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.httpx_mock(assert_all_responses_were_requested=False) +async def test_simulate_component_service_http_call( + httpx_mock: HTTPXMock, monkeypatch: MonkeyPatch +): + monkeypatch.setenv("UIPATH_URL", "https://example.com/myorg/mytenant") + monkeypatch.setenv("UIPATH_ACCESS_TOKEN", "token") + + httpx_mock.add_response( + url="https://example.com/myorg/mytenant/agentsruntime_/api/execution/simulations/simulate-component", + method="POST", + json={"status": 1, "simulatedOutput": "result"}, + ) + + service = _create_simulate_component_service() + assert isinstance(service, SimulateComponentService) + + result = await service.simulate({"componentId": "my_tool"}) + assert result == {"status": 1, "simulatedOutput": "result"} + + +@pytest.mark.asyncio +@pytest.mark.httpx_mock(assert_all_responses_were_requested=False) +async def test_simulate_component_service_no_headers( + httpx_mock: HTTPXMock, monkeypatch: MonkeyPatch +): + monkeypatch.setenv("UIPATH_URL", "https://example.com/myorg/mytenant") + monkeypatch.setenv("UIPATH_ACCESS_TOKEN", "token") + + httpx_mock.add_response( + url="https://example.com/myorg/mytenant/agentsruntime_/api/execution/simulations/simulate-component", + method="POST", + json={"status": 0, "error": {"message": "boom"}}, + ) + + service = _create_simulate_component_service() + result = await service.simulate({"componentId": "my_tool"}) + assert result["status"] == 0 diff --git a/packages/uipath/tests/cli/eval/mocks/test_structured_output.py b/packages/uipath/tests/cli/eval/mocks/test_structured_output.py new file mode 100644 index 000000000..79ad31591 --- /dev/null +++ b/packages/uipath/tests/cli/eval/mocks/test_structured_output.py @@ -0,0 +1,295 @@ +"""Unit tests for the provider-agnostic structured-output helpers.""" + +import json +from types import SimpleNamespace +from typing import Any + +import pytest + +from uipath.eval.mocks._structured_output import ( + RESPONSE_KEY, + RESPONSE_TOOL_NAME, + build_response_tool, + extract_response, + generate_structured_output, +) + + +def _response(message: SimpleNamespace | None) -> SimpleNamespace: + choices = [] if message is None else [SimpleNamespace(message=message)] + return SimpleNamespace(choices=choices) + + +class _FakeLLM: + """Records chat_completions calls and replays queued responses in order.""" + + def __init__(self, responses: list[Any]): + self._responses = list(responses) + self.calls: list[dict[str, Any]] = [] + + async def chat_completions(self, messages: Any, **kwargs: Any) -> Any: + self.calls.append(kwargs) + nxt = self._responses.pop(0) + if isinstance(nxt, Exception): + raise nxt + return nxt + + +def test_build_response_tool_wraps_schema_under_response(): + tool = build_response_tool({"type": "string"}, description="desc") + assert tool["name"] == RESPONSE_TOOL_NAME + assert tool["description"] == "desc" + assert tool["parameters"]["properties"][RESPONSE_KEY] == {"type": "string"} + assert tool["parameters"]["required"] == [RESPONSE_KEY] + + +def test_build_response_tool_inlines_refs_into_self_contained_schema(): + # Nested Pydantic models / enums emit $defs + $ref. The normalized gateway + # accepts $ref/$defs in response_format but NOT in a tool's parameters, so the + # schema must be inlined into a self-contained form (no $ref/$defs anywhere). + operator_def = {"enum": ["+", "-", "*", "/"], "type": "string"} + item_def = {"type": "object", "properties": {"sku": {"type": "string"}}} + schema = { + "type": "object", + "properties": { + "operator": {"$ref": "#/$defs/Operator"}, + "items": {"type": "array", "items": {"$ref": "#/$defs/Item"}}, + }, + "required": ["operator"], + "$defs": {"Operator": operator_def, "Item": item_def}, + } + + tool = build_response_tool(schema, description="d") + params = tool["parameters"] + + blob = json.dumps(params) + assert "$ref" not in blob + assert "$defs" not in blob + + response = params["properties"][RESPONSE_KEY] + assert response["properties"]["operator"] == operator_def + assert response["properties"]["items"]["items"] == item_def + # caller's schema is not mutated + assert "$defs" in schema + + +def test_build_response_tool_keeps_defs_for_cyclic_refs(): + # Self-referential schemas can't be fully inlined; keep $defs hoisted so the + # remaining $ref still resolves rather than infinite-looping. + node_def = { + "type": "object", + "properties": {"child": {"$ref": "#/$defs/Node"}}, + } + schema = { + "type": "object", + "properties": {"root": {"$ref": "#/$defs/Node"}}, + "$defs": {"Node": node_def}, + } + + tool = build_response_tool(schema, description="d") + params = tool["parameters"] + + assert "$defs" in params + assert "$ref" in json.dumps(params) + # the caller's schema dict is not mutated + assert "$defs" in schema + + +def test_extract_response_returns_wrapped_value(): + message = SimpleNamespace( + content=None, + tool_calls=[SimpleNamespace(arguments={RESPONSE_KEY: {"a": 1}})], + ) + assert extract_response(_response(message)) == {"a": 1} + + +def test_extract_response_raises_when_no_choices(): + with pytest.raises(ValueError, match="no choices"): + extract_response(_response(None)) + + +def test_extract_response_raises_when_no_tool_calls(): + # Non-OpenAI text response without a tool call: surface a clear error. + message = SimpleNamespace(content="not a tool call", tool_calls=None) + with pytest.raises(ValueError, match="no tool calls"): + extract_response(_response(message)) + + +def test_extract_response_raises_when_response_key_missing(): + message = SimpleNamespace( + content=None, tool_calls=[SimpleNamespace(arguments={"other": 1})] + ) + with pytest.raises(ValueError, match=RESPONSE_KEY): + extract_response(_response(message)) + + +@pytest.mark.asyncio +async def test_generate_structured_output_prefers_response_format_content(): + # OpenAI returns content via response_format; no fallback call is made. + llm = _FakeLLM([_response(SimpleNamespace(content='{"a": 1}', tool_calls=None))]) + result = await generate_structured_output( + llm, + [{"role": "user", "content": "x"}], + schema={"type": "object"}, + response_format_name="OutputSchema", + description="d", + completion_kwargs={}, + ) + assert result == {"a": 1} + assert len(llm.calls) == 1 + assert "response_format" in llm.calls[0] + assert "tools" not in llm.calls[0] + + +@pytest.mark.asyncio +async def test_generate_structured_output_falls_back_on_prose_content(): + # Claude on the normalized gateway answers response_format requests with + # plain prose (e.g. "Tokyo") — truthy but not JSON. Must fall back to tools + # instead of raising JSONDecodeError (AE-1646). + llm = _FakeLLM( + [ + _response(SimpleNamespace(content="Tokyo", tool_calls=None)), + _response( + SimpleNamespace( + content=None, + tool_calls=[SimpleNamespace(arguments={RESPONSE_KEY: {"a": 1}})], + ) + ), + ] + ) + result = await generate_structured_output( + llm, + [{"role": "user", "content": "x"}], + schema={"type": "object"}, + response_format_name="OutputSchema", + description="d", + completion_kwargs={}, + ) + assert result == {"a": 1} + assert len(llm.calls) == 2 + assert "tools" in llm.calls[1] + + +@pytest.mark.asyncio +async def test_generate_structured_output_falls_back_on_empty_content(): + # Non-OpenAI: response_format yields empty content -> fall back to tool call. + llm = _FakeLLM( + [ + _response(SimpleNamespace(content=None, tool_calls=None)), + _response( + SimpleNamespace( + content=None, + tool_calls=[SimpleNamespace(arguments={RESPONSE_KEY: {"a": 1}})], + ) + ), + ] + ) + result = await generate_structured_output( + llm, + [{"role": "user", "content": "x"}], + schema={"type": "object"}, + response_format_name="OutputSchema", + description="d", + completion_kwargs={}, + ) + assert result == {"a": 1} + assert len(llm.calls) == 2 + assert "response_format" in llm.calls[0] + assert "tools" in llm.calls[1] and "tool_choice" in llm.calls[1] + + +@pytest.mark.asyncio +async def test_generate_structured_output_falls_back_when_response_format_raises(): + # A provider that rejects response_format outright still gets a tool fallback. + llm = _FakeLLM( + [ + RuntimeError("response_format unsupported"), + _response( + SimpleNamespace( + content=None, + tool_calls=[SimpleNamespace(arguments={RESPONSE_KEY: "ok"})], + ) + ), + ] + ) + result = await generate_structured_output( + llm, + [{"role": "user", "content": "x"}], + schema={"type": "string"}, + response_format_name="OutputSchema", + description="d", + completion_kwargs={}, + ) + assert result == "ok" + assert len(llm.calls) == 2 + + +def test_build_response_tool_merges_ref_sibling_keys(): + # Pydantic can emit sibling keys (e.g. description) next to $ref; they + # must survive inlining since they guide the LLM. + schema = { + "type": "object", + "properties": { + "op": {"$ref": "#/$defs/Op", "description": "the operator to use"} + }, + "$defs": {"Op": {"type": "string", "enum": ["+", "-"]}}, + } + tool = build_response_tool(schema, description="d") + op = tool["parameters"]["properties"][RESPONSE_KEY]["properties"]["op"] + assert op == { + "type": "string", + "enum": ["+", "-"], + "description": "the operator to use", + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model", + [ + "anthropic.claude-sonnet-4-5-20250929-v1:0", + "claude-haiku-4-5", + "gemini-2.5-pro", + ], +) +async def test_non_openai_models_use_tool_call_directly(model: str): + # Claude/Gemini don't honor response_format on the normalized gateway, so + # their strategies skip it entirely: a single forced tool call. + llm = _FakeLLM( + [ + _response( + SimpleNamespace( + content=None, + tool_calls=[SimpleNamespace(arguments={RESPONSE_KEY: "ok"})], + ) + ) + ] + ) + result = await generate_structured_output( + llm, + [{"role": "user", "content": "x"}], + schema={"type": "string"}, + response_format_name="OutputSchema", + description="d", + completion_kwargs={"model": model}, + ) + assert result == "ok" + assert len(llm.calls) == 1 + assert "tools" in llm.calls[0] and "tool_choice" in llm.calls[0] + assert "response_format" not in llm.calls[0] + + +@pytest.mark.asyncio +async def test_openai_models_prefer_response_format(): + llm = _FakeLLM([_response(SimpleNamespace(content='{"a": 1}', tool_calls=None))]) + result = await generate_structured_output( + llm, + [{"role": "user", "content": "x"}], + schema={"type": "object"}, + response_format_name="OutputSchema", + description="d", + completion_kwargs={"model": "gpt-4.1-mini-2025-04-14"}, + ) + assert result == {"a": 1} + assert len(llm.calls) == 1 + assert "response_format" in llm.calls[0] diff --git a/packages/uipath/tests/cli/eval/test_agent_memory_settings.py b/packages/uipath/tests/cli/eval/test_agent_memory_settings.py new file mode 100644 index 000000000..11a71c58e --- /dev/null +++ b/packages/uipath/tests/cli/eval/test_agent_memory_settings.py @@ -0,0 +1,257 @@ +"""Tests for agent memory settings in evaluation sets and the eval CLI override.""" + +import json +import tempfile +from pathlib import Path + +from uipath._cli.cli_eval import _resolve_agent_memory_settings_override +from uipath.eval.helpers import EvalHelpers +from uipath.eval.models.evaluation_set import ( + EvaluationSet, + EvaluationSetAgentMemorySettings, +) + + +def make_eval_set(**overrides) -> EvaluationSet: + data = { + "id": "eval-set-1", + "name": "Test Eval Set", + "version": "1.0", + "evaluatorConfigs": [], + "evaluations": [], + **overrides, + } + return EvaluationSet.model_validate(data) + + +class TestEvaluationSetSchema: + def test_parses_agent_memory_fields(self): + eval_set = make_eval_set( + agentMemoryEnabled=True, + agentMemorySettings=[ + { + "id": "primary-memory", + "resultCount": "5", + "searchMode": "hybrid", + "threshold": "0.8", + } + ], + ) + + assert eval_set.agent_memory_enabled is True + assert len(eval_set.agent_memory_settings) == 1 + setting = eval_set.agent_memory_settings[0] + assert setting.id == "primary-memory" + assert setting.result_count == "5" + assert setting.search_mode == "hybrid" + assert setting.threshold == "0.8" + + def test_defaults_preserve_existing_eval_sets(self): + eval_set = make_eval_set() + + assert eval_set.agent_memory_enabled is False + assert eval_set.agent_memory_settings == [] + + def test_memory_settings_default_to_same_as_agent(self): + setting = EvaluationSetAgentMemorySettings(id="s1") + + assert setting.result_count == "same-as-agent" + assert setting.search_mode == "same-as-agent" + assert setting.threshold == "same-as-agent" + + def test_serializes_with_camel_case_aliases(self): + eval_set = make_eval_set( + agentMemoryEnabled=True, + agentMemorySettings=[{"id": "s1", "searchMode": "semantic"}], + ) + + dumped = eval_set.model_dump(by_alias=True) + assert dumped["agentMemoryEnabled"] is True + assert dumped["agentMemorySettings"][0]["searchMode"] == "semantic" + + +class TestLegacyEvalSetMigration: + def test_legacy_migration_preserves_agent_memory_fields(self): + legacy_eval_set = { + "fileName": "test-eval.json", + "id": "test-eval-set-id", + "name": "Test Eval Set", + "batchSize": 10, + "evaluatorRefs": ["evaluator1"], + "evaluations": [], + "modelSettings": [], + "agentMemoryEnabled": True, + "agentMemorySettings": [ + { + "id": "primary-memory", + "resultCount": "5", + "searchMode": "hybrid", + "threshold": "0.8", + } + ], + "createdAt": "2025-01-26T00:00:00.000Z", + "updatedAt": "2025-01-26T00:00:00.000Z", + } + + with tempfile.TemporaryDirectory() as tmpdir: + eval_file = Path(tmpdir) / "eval-set.json" + eval_file.write_text(json.dumps(legacy_eval_set)) + + loaded_eval_set, _ = EvalHelpers.load_eval_set(str(eval_file)) + + assert loaded_eval_set.agent_memory_enabled is True + assert len(loaded_eval_set.agent_memory_settings) == 1 + assert loaded_eval_set.agent_memory_settings[0].id == "primary-memory" + + +class TestResolveAgentMemorySettingsOverride: + def test_memory_disabled_returns_disabled_override(self): + eval_set = make_eval_set(agentMemoryEnabled=False) + + override = _resolve_agent_memory_settings_override("default", eval_set) + + assert override == {"enabled": False} + + def test_memory_enabled_default_id_uses_first_setting(self): + eval_set = make_eval_set( + agentMemoryEnabled=True, + agentMemorySettings=[ + { + "id": "s1", + "resultCount": "5", + "searchMode": "hybrid", + "threshold": "0.8", + }, + { + "id": "s2", + "resultCount": "10", + "searchMode": "semantic", + "threshold": "0.5", + }, + ], + ) + + override = _resolve_agent_memory_settings_override("default", eval_set) + + assert override == { + "enabled": True, + "resultCount": "5", + "searchMode": "hybrid", + "threshold": "0.8", + } + + def test_memory_enabled_selects_setting_by_id(self): + eval_set = make_eval_set( + agentMemoryEnabled=True, + agentMemorySettings=[ + {"id": "s1", "searchMode": "hybrid"}, + { + "id": "s2", + "resultCount": "10", + "searchMode": "semantic", + "threshold": "0.5", + }, + ], + ) + + override = _resolve_agent_memory_settings_override("s2", eval_set) + + assert override == { + "enabled": True, + "resultCount": "10", + "searchMode": "semantic", + "threshold": "0.5", + } + + def test_default_id_matches_persisted_default_entry(self): + # The eval-set editor persists a "default" entry (all same-as-agent) + # alongside user-defined settings; it may not be first in the list. + eval_set = make_eval_set( + agentMemoryEnabled=True, + agentMemorySettings=[ + { + "id": "s1", + "resultCount": "10", + "searchMode": "semantic", + "threshold": "0.5", + }, + {"id": "default"}, + ], + ) + + override = _resolve_agent_memory_settings_override("default", eval_set) + + assert override == { + "enabled": True, + "resultCount": "same-as-agent", + "searchMode": "same-as-agent", + "threshold": "same-as-agent", + } + + def test_unknown_id_falls_back_to_first_setting(self): + eval_set = make_eval_set( + agentMemoryEnabled=True, + agentMemorySettings=[{"id": "s1", "searchMode": "hybrid"}], + ) + + override = _resolve_agent_memory_settings_override("missing", eval_set) + + assert override["enabled"] is True + assert override["searchMode"] == "hybrid" + + def test_memory_enabled_without_settings_keeps_agent_configuration(self): + eval_set = make_eval_set(agentMemoryEnabled=True) + + override = _resolve_agent_memory_settings_override("default", eval_set) + + assert override == {"enabled": True} + + def test_no_memory_id_disables_memory(self): + # Selecting "No memory" in the eval settings passes the "NoMemory" + # sentinel id; its entry stores "NoMemory" field values that must not + # be applied as real settings. + eval_set = make_eval_set( + agentMemoryEnabled=True, + agentMemorySettings=[ + {"id": "s1", "searchMode": "hybrid"}, + { + "id": "NoMemory", + "resultCount": "NoMemory", + "searchMode": "semantic", + "threshold": "NoMemory", + }, + ], + ) + + override = _resolve_agent_memory_settings_override("NoMemory", eval_set) + + assert override == {"enabled": False} + + def test_no_memory_id_disables_memory_without_matching_entry(self): + # The sentinel disables memory even when the eval set has no + # "NoMemory" entry; it must not fall back to the first setting. + eval_set = make_eval_set( + agentMemoryEnabled=True, + agentMemorySettings=[{"id": "s1", "searchMode": "hybrid"}], + ) + + override = _resolve_agent_memory_settings_override("NoMemory", eval_set) + + assert override == {"enabled": False} + + def test_fallback_to_no_memory_entry_disables_memory(self): + eval_set = make_eval_set( + agentMemoryEnabled=True, + agentMemorySettings=[ + { + "id": "NoMemory", + "resultCount": "NoMemory", + "searchMode": "semantic", + "threshold": "NoMemory", + } + ], + ) + + override = _resolve_agent_memory_settings_override("default", eval_set) + + assert override == {"enabled": False} diff --git a/packages/uipath/tests/cli/eval/test_eval_id_casing.py b/packages/uipath/tests/cli/eval/test_eval_id_casing.py new file mode 100644 index 000000000..e14a88a68 --- /dev/null +++ b/packages/uipath/tests/cli/eval/test_eval_id_casing.py @@ -0,0 +1,69 @@ +"""Tests for case-insensitive eval id handling (PC-4688). + +Eval sets exported by some tools emit uppercase GUID ids. The backend +canonicalizes GUIDs to lowercase, so any case-sensitive correlation on the +runtime side (selection, span/cache keying) silently fails to match. These +tests pin the fix: GUID ids are normalized to lowercase at ingestion and +selection is casing-agnostic. +""" + +from typing import Any + +from uipath.eval.models.evaluation_set import ( + EvaluationItem, + EvaluationSet, + LegacyEvaluationItem, +) + +UPPER_GUID = "B063907C-76AB-4B0A-88A3-EC0FB40698B8" +LOWER_GUID = "b063907c-76ab-4b0a-88a3-ec0fb40698b8" + + +def _make_item(eval_id: str) -> dict[str, Any]: + return { + "id": eval_id, + "name": "item", + "inputs": {"x": 1}, + "evaluationCriterias": {}, + } + + +def test_evaluation_item_normalizes_uppercase_guid_id(): + """An uppercase GUID id is stored in canonical lowercase form.""" + item = EvaluationItem.model_validate(_make_item(UPPER_GUID)) + assert item.id == LOWER_GUID + + +def test_legacy_evaluation_item_normalizes_uppercase_guid_id(): + """LegacyEvaluationItem also normalizes uppercase GUID ids.""" + item = LegacyEvaluationItem.model_validate( + { + "id": UPPER_GUID, + "name": "item", + "inputs": {"x": 1}, + "expectedOutput": {}, + "evalSetId": "set-1", + "createdAt": "2025-01-01T00:00:00.000Z", + "updatedAt": "2025-01-01T00:00:00.000Z", + } + ) + assert item.id == LOWER_GUID + + +def test_non_guid_id_is_left_unchanged(): + """Non-GUID ids (e.g. slugs) keep their original value and casing.""" + item = EvaluationItem.model_validate(_make_item("Test-Eval-1")) + assert item.id == "Test-Eval-1" + + +def test_extract_selected_evals_matches_regardless_of_caller_casing(): + """Selecting by an uppercase GUID matches a normalized stored id.""" + eval_set = EvaluationSet.model_validate( + { + "id": "set-1", + "name": "set", + "evaluations": [_make_item(LOWER_GUID), _make_item("other-id")], + } + ) + eval_set.extract_selected_evals([UPPER_GUID]) + assert [e.id for e in eval_set.evaluations] == [LOWER_GUID] diff --git a/packages/uipath/tests/cli/eval/test_eval_resource_overwrites.py b/packages/uipath/tests/cli/eval/test_eval_resource_overwrites.py new file mode 100644 index 000000000..71589fb9a --- /dev/null +++ b/packages/uipath/tests/cli/eval/test_eval_resource_overwrites.py @@ -0,0 +1,185 @@ +"""Tests for resource overwrites context ordering in the eval CLI. + +The overwrites context must be entered before the runtime is created: +building the agent graph resolves folder-scoped resources (e.g. escalation +memory spaces) at tool-creation time, and those lookups need the +overwritten folder paths. +""" + +import json +import os +from contextlib import ExitStack +from typing import Any +from unittest.mock import AsyncMock, Mock, patch + +import pytest +from click.testing import CliRunner + +from uipath._cli import cli +from uipath._cli.middlewares import MiddlewareResult +from uipath.platform.common._bindings import _resource_overwrites + + +def _middleware_continue() -> MiddlewareResult: + return MiddlewareResult( + should_continue=True, + error_message=None, + should_include_stacktrace=False, + ) + + +def _write_project_files() -> None: + with open("uipath.json", "w") as f: + json.dump({"functions": {"agent": "main.py:main"}}, f) + + os.makedirs("evaluations/eval-sets", exist_ok=True) + eval_set = { + "version": "1.0", + "id": "test-set", + "name": "Test Set", + "evaluatorRefs": [], + "evaluations": [], + } + with open("evaluations/eval-sets/test-set.json", "w") as f: + json.dump(eval_set, f) + + +def _make_mock_runtime() -> Mock: + mock_runtime = Mock() + mock_runtime.get_schema = AsyncMock( + return_value=Mock(metadata=None, input_schema=None, output_schema=None) + ) + mock_runtime.dispose = AsyncMock() + return mock_runtime + + +def _make_mock_factory(mock_runtime: Mock) -> Mock: + mock_factory = Mock() + mock_factory.discover_entrypoints.return_value = ["agent"] + mock_factory.get_settings = AsyncMock(return_value=None) + mock_factory.dispose = AsyncMock() + mock_factory.new_runtime = AsyncMock(return_value=mock_runtime) + return mock_factory + + +def _enter_base_patches(stack: ExitStack, mock_factory: Mock) -> None: + stack.enter_context( + patch( + "uipath._cli.cli_eval.Middlewares.next", + return_value=_middleware_continue(), + ) + ) + stack.enter_context( + patch( + "uipath._cli.cli_eval.UiPathRuntimeFactoryRegistry.get", + return_value=mock_factory, + ) + ) + stack.enter_context( + patch("uipath._cli.cli_eval.setup_reporting_prereq", return_value=False) + ) + stack.enter_context( + patch( + "uipath._cli.cli_eval.EvalHelpers.load_evaluators", + new=AsyncMock(return_value=[]), + ) + ) + stack.enter_context( + patch("uipath._cli.cli_eval.evaluate", new=AsyncMock(return_value=None)) + ) + + +class TestEvalResourceOverwritesOrdering: + def test_overwrites_context_active_when_runtime_is_created( + self, runner: CliRunner, temp_dir: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + """new_runtime must run inside the resource overwrites context.""" + monkeypatch.setenv("UIPATH_PROJECT_ID", "project-123") + + with runner.isolated_filesystem(temp_dir=temp_dir): + _write_project_files() + + overwrite = Mock() + overwrites = {"memorySpace.MemorySpace": overwrite} + overwrites_seen_by_new_runtime: list[Any] = [] + + mock_runtime = _make_mock_runtime() + + async def new_runtime(*args: Any, **kwargs: Any) -> Mock: + overwrites_seen_by_new_runtime.append(_resource_overwrites.get()) + return mock_runtime + + mock_factory = _make_mock_factory(mock_runtime) + mock_factory.new_runtime = AsyncMock(side_effect=new_runtime) + + mock_studio_client = Mock() + mock_studio_client.get_resource_overwrites = AsyncMock( + return_value=overwrites + ) + + with ExitStack() as stack: + _enter_base_patches(stack, mock_factory) + stack.enter_context( + patch( + "uipath._cli.cli_eval.StudioClient", + return_value=mock_studio_client, + ) + ) + result = runner.invoke(cli, ["eval"]) + + assert result.exit_code == 0 + mock_studio_client.get_resource_overwrites.assert_awaited_once() + assert overwrites_seen_by_new_runtime == [overwrites] + mock_runtime.dispose.assert_awaited_once() + + def test_no_project_id_runs_without_overwrites( + self, runner: CliRunner, temp_dir: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("UIPATH_PROJECT_ID", raising=False) + + with runner.isolated_filesystem(temp_dir=temp_dir): + _write_project_files() + + overwrites_seen_by_new_runtime: list[Any] = [] + mock_runtime = _make_mock_runtime() + + async def new_runtime(*args: Any, **kwargs: Any) -> Mock: + overwrites_seen_by_new_runtime.append(_resource_overwrites.get()) + return mock_runtime + + mock_factory = _make_mock_factory(mock_runtime) + mock_factory.new_runtime = AsyncMock(side_effect=new_runtime) + + with ExitStack() as stack: + _enter_base_patches(stack, mock_factory) + mock_studio_client_cls = stack.enter_context( + patch("uipath._cli.cli_eval.StudioClient") + ) + result = runner.invoke(cli, ["eval"]) + + assert result.exit_code == 0 + mock_studio_client_cls.assert_not_called() + assert overwrites_seen_by_new_runtime == [None] + mock_runtime.dispose.assert_awaited_once() + + def test_runtime_disposed_when_schema_loading_fails( + self, runner: CliRunner, temp_dir: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("UIPATH_PROJECT_ID", raising=False) + + with runner.isolated_filesystem(temp_dir=temp_dir): + _write_project_files() + + mock_runtime = _make_mock_runtime() + mock_runtime.get_schema = AsyncMock( + side_effect=RuntimeError("schema loading failed") + ) + mock_factory = _make_mock_factory(mock_runtime) + + with ExitStack() as stack: + _enter_base_patches(stack, mock_factory) + stack.enter_context(patch("uipath._cli.cli_eval.StudioClient")) + result = runner.invoke(cli, ["eval"]) + + assert result.exit_code != 0 + mock_runtime.dispose.assert_awaited_once() diff --git a/packages/uipath/tests/cli/eval/test_eval_runtime_metadata.py b/packages/uipath/tests/cli/eval/test_eval_runtime_metadata.py index 112f8774b..07042cc12 100644 --- a/packages/uipath/tests/cli/eval/test_eval_runtime_metadata.py +++ b/packages/uipath/tests/cli/eval/test_eval_runtime_metadata.py @@ -1,7 +1,7 @@ """Tests for UiPathEvalRuntime metadata loading functionality. This module tests: -- _get_agent_model() - cached agent model retrieval +- get_agent_model() - cached agent model retrieval - get_schema() - cached schema retrieval """ @@ -10,11 +10,9 @@ import pytest -from uipath._cli.cli_eval import ( - _get_agent_model, -) from uipath.core.events import EventBus from uipath.core.tracing import UiPathTraceManager +from uipath.eval.helpers import get_agent_model from uipath.eval.runtime import UiPathEvalContext, UiPathEvalRuntime from uipath.runtime import ( UiPathExecuteOptions, @@ -119,34 +117,34 @@ async def dispose(self) -> None: class TestGetAgentModel: - """Tests for _get_agent_model function.""" + """Tests for get_agent_model function.""" @pytest.mark.asyncio async def test_returns_agent_model(self): - """Test that _get_agent_model returns the correct model from schema.""" + """Test that get_agent_model returns the correct model from schema.""" schema = MockRuntimeSchema() schema.metadata = {"settings": {"model": "gpt-4o-2024-11-20"}} - model = _get_agent_model(schema) + model = get_agent_model(schema) assert model == "gpt-4o-2024-11-20" @pytest.mark.asyncio async def test_returns_none_when_no_model(self): - """Test that _get_agent_model returns None when runtime has no model.""" + """Test that get_agent_model returns None when runtime has no model.""" schema = MockRuntimeSchema() - model = _get_agent_model(schema) + model = get_agent_model(schema) assert model is None @pytest.mark.asyncio async def test_returns_model_consistently(self): - """Test that _get_agent_model returns consistent results.""" + """Test that get_agent_model returns consistent results.""" schema = MockRuntimeSchema() schema.metadata = {"settings": {"model": "consistent-model"}} # Multiple calls should return the same value - model1 = _get_agent_model(schema) - model2 = _get_agent_model(schema) + model1 = get_agent_model(schema) + model2 = get_agent_model(schema) assert model1 == model2 == "consistent-model" diff --git a/packages/uipath/tests/cli/eval/test_eval_span_utils.py b/packages/uipath/tests/cli/eval/test_eval_span_utils.py index c6b48b0b6..1205a5d82 100644 --- a/packages/uipath/tests/cli/eval/test_eval_span_utils.py +++ b/packages/uipath/tests/cli/eval/test_eval_span_utils.py @@ -460,16 +460,16 @@ async def test_configure_evaluation_span(self): @pytest.mark.asyncio async def test_configure_evaluation_span_with_error(self): - """Test configuring evaluation span when agent execution has error.""" + """Test configuring evaluation span when workload execution has error.""" span = MockSpan() # Mock evaluation run results (empty since agent failed) mock_evaluation_run_results = MagicMock() mock_evaluation_run_results.evaluation_run_results = [] - # Mock agent execution output with error + # Mock workload execution output with error mock_agent_output = MagicMock() - mock_agent_output.result.error = "Agent execution failed" + mock_agent_output.result.error = "Workload execution failed" await configure_evaluation_span( span=span, # type: ignore[arg-type] @@ -487,11 +487,11 @@ async def test_configure_evaluation_span_with_error(self): assert span._status is not None assert span._status.status_code == StatusCode.ERROR assert span._status.description is not None - assert "Agent execution failed" in span._status.description + assert "Workload execution failed" in span._status.description @pytest.mark.asyncio async def test_configure_evaluation_span_without_agent_output(self): - """Test configuring evaluation span without agent execution output.""" + """Test configuring evaluation span without workload execution output.""" span = MockSpan() mock_result = MagicMock() diff --git a/packages/uipath/tests/cli/eval/test_eval_telemetry.py b/packages/uipath/tests/cli/eval/test_eval_telemetry.py index fdfe7135c..fdcaf4a85 100644 --- a/packages/uipath/tests/cli/eval/test_eval_telemetry.py +++ b/packages/uipath/tests/cli/eval/test_eval_telemetry.py @@ -25,6 +25,7 @@ EvalSetRunCreatedEvent, EvalSetRunUpdatedEvent, ) +from uipath.platform.constants import ENV_UIPATH_AGENT_ID class TestEventNameConstants: @@ -95,6 +96,7 @@ def _create_eval_set_run_created_event( entrypoint: str = "agent.py", no_of_evals: int = 5, evaluators: list[Any] | None = None, + agent_type: str | None = None, ) -> EvalSetRunCreatedEvent: """Helper to create EvalSetRunCreatedEvent.""" return EvalSetRunCreatedEvent( @@ -103,9 +105,60 @@ def _create_eval_set_run_created_event( eval_set_run_id=eval_set_run_id, entrypoint=entrypoint, no_of_evals=no_of_evals, + agent_type=agent_type, evaluators=evaluators or [], ) + @pytest.mark.asyncio + @patch("uipath._cli._evals._telemetry.track_event") + async def test_agent_type_normalized_to_wire_lowcode(self, mock_track_event): + """Modern factory labels containing ``lowcode`` map to ``"LowCode"`` + so Application Insights dashboards that filter on the historical + wire value keep working after the factory-supplied refactor. + """ + subscriber = EvalTelemetrySubscriber() + event = self._create_eval_set_run_created_event(agent_type="uipath_lowcode") + + await subscriber._on_eval_set_run_created(event) + + properties = mock_track_event.call_args[0][1] + assert properties["AgentType"] == "LowCode" + + @pytest.mark.asyncio + @patch("uipath._cli._evals._telemetry.track_event") + async def test_agent_type_normalized_to_wire_coded(self, mock_track_event): + """Modern factory labels that aren't low-code map to ``"Coded"``.""" + subscriber = EvalTelemetrySubscriber() + event = self._create_eval_set_run_created_event(agent_type="uipath_coded") + + await subscriber._on_eval_set_run_created(event) + + properties = mock_track_event.call_args[0][1] + assert properties["AgentType"] == "Coded" + + @pytest.mark.asyncio + @patch("uipath._cli._evals._telemetry.track_event") + async def test_agent_type_falls_back_to_entrypoint_when_missing( + self, mock_track_event + ): + """Pre-refactor callers didn't set ``agent_type``; keep the + ``agent.json`` ⇒ ``"LowCode"`` derivation so no in-flight consumer + breaks. Anything else falls back to ``"Coded"``. + """ + subscriber = EvalTelemetrySubscriber() + + low_code_event = self._create_eval_set_run_created_event( + agent_type=None, entrypoint="agent.json" + ) + await subscriber._on_eval_set_run_created(low_code_event) + assert mock_track_event.call_args[0][1]["AgentType"] == "LowCode" + + coded_event = self._create_eval_set_run_created_event( + agent_type=None, entrypoint="agent.py" + ) + await subscriber._on_eval_set_run_created(coded_event) + assert mock_track_event.call_args[0][1]["AgentType"] == "Coded" + @pytest.mark.asyncio @patch("uipath._cli._evals._telemetry.track_event") async def test_on_eval_set_run_created_tracks_event(self, mock_track_event): @@ -284,6 +337,45 @@ async def test_on_eval_run_updated_failure(self, mock_track_event): assert "Test error" in properties["ErrorMessage"] assert properties["IsRuntimeException"] is True + @pytest.mark.asyncio + @patch("uipath._cli._evals._telemetry.track_event") + async def test_on_eval_run_updated_user_error_includes_classification( + self, mock_track_event + ): + """User-category runtime errors emit ErrorCode/ErrorCategory and are not runtime exceptions.""" + from uipath.runtime.errors import ( + UiPathErrorCategory, + UiPathErrorCode, + UiPathRuntimeError, + ) + + subscriber = EvalTelemetrySubscriber() + error = UiPathRuntimeError( + UiPathErrorCode.INPUT_INVALID_JSON, + "Invalid input", + "Input does not match the expected schema", + UiPathErrorCategory.USER, + include_traceback=False, + ) + exception_details = EvalItemExceptionDetails( + exception=error, + runtime_exception=False, + ) + event = self._create_eval_run_updated_event( + success=False, + exception_details=exception_details, + ) + + await subscriber._on_eval_run_updated(event) + + call_args = mock_track_event.call_args + assert call_args[0][0] == EVAL_RUN_FAILED + properties = call_args[0][1] + assert properties["ErrorType"] == "UiPathRuntimeError" + assert properties["ErrorCode"] == "Python.INPUT_INVALID_JSON" + assert properties["ErrorCategory"] == "User" + assert properties["IsRuntimeException"] is False + @pytest.mark.asyncio @patch("uipath._cli._evals._telemetry.track_event") async def test_on_eval_run_updated_with_scores(self, mock_track_event): @@ -312,7 +404,7 @@ async def test_on_eval_run_updated_with_scores(self, mock_track_event): async def test_on_eval_run_updated_agent_execution_time_converted_to_ms( self, mock_track_event ): - """Test that agent execution time is converted to milliseconds.""" + """Test that workload execution time is converted to milliseconds.""" subscriber = EvalTelemetrySubscriber() event = self._create_eval_run_updated_event(agent_execution_time=2.5) @@ -422,6 +514,10 @@ def test_enrich_properties_adds_env_vars(self, mock_get_claim): """Test that environment variables are added when present.""" mock_get_claim.return_value = "user-789" + from uipath.platform.common._span_utils import _read_config_id + + _read_config_id.cache_clear() + subscriber = EvalTelemetrySubscriber() properties: dict[str, Any] = {} @@ -429,23 +525,30 @@ def test_enrich_properties_adds_env_vars(self, mock_get_claim): os.environ, { "UIPATH_PROJECT_ID": "project-123", + ENV_UIPATH_AGENT_ID: "agent-123", "UIPATH_ORGANIZATION_ID": "org-456", "UIPATH_TENANT_ID": "tenant-abc", + "UIPATH_EVAL_RUN_SOURCE": "FirstSuccessfulRun", }, ): subscriber._enrich_properties(properties) assert properties["ProjectId"] == "project-123" - assert properties["AgentId"] == "project-123" + assert properties["AgentId"] == "agent-123" assert properties["CloudOrganizationId"] == "org-456" assert properties["CloudUserId"] == "user-789" assert properties["TenantId"] == "tenant-abc" + assert properties["RunSource"] == "FirstSuccessfulRun" @patch("uipath._cli._evals._telemetry.get_claim_from_token") def test_enrich_properties_skips_missing_env_vars(self, mock_get_claim): """Test that missing environment variables are not added.""" mock_get_claim.side_effect = Exception("No token") + from uipath.platform.common._span_utils import _read_config_id + + _read_config_id.cache_clear() + subscriber = EvalTelemetrySubscriber() properties: dict[str, Any] = {} @@ -453,8 +556,10 @@ def test_enrich_properties_skips_missing_env_vars(self, mock_get_claim): # Remove env vars if they exist for key in [ "UIPATH_PROJECT_ID", + ENV_UIPATH_AGENT_ID, "UIPATH_ORGANIZATION_ID", "UIPATH_TENANT_ID", + "UIPATH_EVAL_RUN_SOURCE", ]: os.environ.pop(key, None) @@ -465,6 +570,7 @@ def test_enrich_properties_skips_missing_env_vars(self, mock_get_claim): assert "CloudOrganizationId" not in properties assert "CloudUserId" not in properties assert "TenantId" not in properties + assert "RunSource" not in properties class TestExceptionHandling: diff --git a/packages/uipath/tests/cli/eval/test_live_tracking_span_processor.py b/packages/uipath/tests/cli/eval/test_live_tracking_span_processor.py index a20c18d9b..d26e89985 100644 --- a/packages/uipath/tests/cli/eval/test_live_tracking_span_processor.py +++ b/packages/uipath/tests/cli/eval/test_live_tracking_span_processor.py @@ -3,7 +3,7 @@ import threading import time from typing import Any -from unittest.mock import Mock +from unittest.mock import Mock, patch import pytest from opentelemetry import context as context_api @@ -348,6 +348,35 @@ def test_shutdown_waits_for_pending_tasks(self, mock_exporter): # Verify executor is shutdown (calling shutdown multiple times should be safe) processor.shutdown() # Should not raise + def test_shutdown_closes_exporter_after_pending_tasks(self, mock_exporter): + calls = [] + + def upsert(*args, **kwargs): + calls.append("upsert") + + def shutdown(): + calls.append("shutdown") + + mock_exporter.upsert_span = Mock(side_effect=upsert) + mock_exporter.shutdown = Mock(side_effect=shutdown) + processor = LiveTrackingSpanProcessor(mock_exporter, max_workers=1) + span = self.create_mock_span({"span_type": "eval"}) + + processor.on_start(span, None) + processor.shutdown() + + assert calls == ["upsert", "shutdown"] + + def test_shutdown_logs_exporter_shutdown_failure(self, mock_exporter): + mock_exporter.shutdown = Mock(side_effect=RuntimeError("close failed")) + processor = LiveTrackingSpanProcessor(mock_exporter) + + with patch("uipath.tracing._live_tracking_processor.logger.debug") as debug: + processor.shutdown() + + mock_exporter.shutdown.assert_called_once_with() + debug.assert_called_once_with("Exporter shutdown failed: close failed") + def test_multiple_processors_independent_thread_pools(self, mock_exporter): """Test that multiple processors have independent thread pools.""" processor1 = LiveTrackingSpanProcessor(mock_exporter, max_workers=5) diff --git a/packages/uipath/tests/cli/eval/test_progress_reporter.py b/packages/uipath/tests/cli/eval/test_progress_reporter.py index 87919c2b3..1fd00bf12 100644 --- a/packages/uipath/tests/cli/eval/test_progress_reporter.py +++ b/packages/uipath/tests/cli/eval/test_progress_reporter.py @@ -927,4 +927,207 @@ def test_build_evaluator_snapshot_skips_non_string_model(self, progress_reporter snapshot = progress_reporter._build_evaluator_snapshot(evaluator) assert snapshot["prompt"] == "Evaluate this" - assert "model" not in snapshot + + +class TestAgentIdRouting: + """Eval-set/eval-run API URLs route by AgentId, not file-source project. + + For local-workspace eval runs the file-source project (UIPATH_PROJECT_ID, + typically the cloud debug project's GUID) differs from the logical agent + (UIPATH_AGENT_ID). The route URL must reflect the logical agent so backend + auth/ownership/telemetry don't see the per-run debug project as the agent. + File fetching (UiPathConfig.project_id) is unaffected. + """ + + def _make_reporter(self, monkeypatch, project_id, agent_id): + monkeypatch.setenv("UIPATH_URL", "https://test.uipath.com") + monkeypatch.setenv("UIPATH_ACCESS_TOKEN", "test-token") + monkeypatch.setenv("UIPATH_TENANT_ID", "test-tenant-id") + if project_id is not None: + monkeypatch.setenv("UIPATH_PROJECT_ID", project_id) + else: + monkeypatch.delenv("UIPATH_PROJECT_ID", raising=False) + if agent_id is not None: + monkeypatch.setenv("UIPATH_AGENT_ID", agent_id) + else: + monkeypatch.delenv("UIPATH_AGENT_ID", raising=False) + return StudioWebProgressReporter() + + def test_agent_id_used_in_url_when_both_set(self, monkeypatch): + reporter = self._make_reporter( + monkeypatch, project_id="debug-project-guid", agent_id="real-agent-id" + ) + assert reporter._agent_id == "real-agent-id" + assert reporter._project_id == "debug-project-guid" + + from uipath._cli._evals._progress_reporter import StudioWebAgentSnapshot + + spec = reporter._create_eval_set_run_spec( + eval_set_id="test-eval-set", + agent_snapshot=StudioWebAgentSnapshot( + input_schema={"type": "object"}, output_schema={"type": "object"} + ), + no_of_evals=1, + is_coded=False, + ) + assert "/agents/real-agent-id/" in spec.endpoint + assert "/agents/debug-project-guid/" not in spec.endpoint + + def test_agent_id_in_eval_set_run_payload(self, monkeypatch): + reporter = self._make_reporter( + monkeypatch, project_id="debug-project-guid", agent_id="real-agent-id" + ) + + from uipath._cli._evals._progress_reporter import StudioWebAgentSnapshot + + spec = reporter._create_eval_set_run_spec( + eval_set_id="test-eval-set", + agent_snapshot=StudioWebAgentSnapshot( + input_schema={"type": "object"}, output_schema={"type": "object"} + ), + no_of_evals=1, + is_coded=False, + ) + assert spec.json["agentId"] == "real-agent-id" + + def test_falls_back_to_project_id_when_agent_id_unset(self, monkeypatch): + reporter = self._make_reporter( + monkeypatch, project_id="cloud-project-id", agent_id=None + ) + assert reporter._agent_id == "cloud-project-id" + + from uipath._cli._evals._progress_reporter import StudioWebAgentSnapshot + + spec = reporter._create_eval_set_run_spec( + eval_set_id="test-eval-set", + agent_snapshot=StudioWebAgentSnapshot( + input_schema={"type": "object"}, output_schema={"type": "object"} + ), + no_of_evals=1, + is_coded=False, + ) + assert "/agents/cloud-project-id/" in spec.endpoint + + +class TestProjectFilesSourcePropagation: + """Reporter must propagate UIPATH_PROJECT_FILES_SOURCE to backend rows. + + Backend filters listings by `projectFilesSource` (Local=1, Cloud=0). Without + the SDK setting it on POST/PUT payloads and GET query params, every row + lands as Cloud and the UI's `?projectFilesSource=1` filter never matches + local-workspace runs. + """ + + def _make_reporter(self, monkeypatch, project_files_source): + monkeypatch.setenv("UIPATH_URL", "https://test.uipath.com") + monkeypatch.setenv("UIPATH_ACCESS_TOKEN", "test-token") + monkeypatch.setenv("UIPATH_TENANT_ID", "test-tenant-id") + monkeypatch.setenv("UIPATH_PROJECT_ID", "test-project-id") + if project_files_source is not None: + monkeypatch.setenv("UIPATH_PROJECT_FILES_SOURCE", project_files_source) + else: + monkeypatch.delenv("UIPATH_PROJECT_FILES_SOURCE", raising=False) + return StudioWebProgressReporter() + + @pytest.mark.parametrize( + "raw,expected", + [("Local", 1), ("local", 1), ("Cloud", 0), ("cloud", 0), ("1", 1), ("0", 0)], + ) + def test_resolves_env_var_to_int(self, monkeypatch, raw, expected): + reporter = self._make_reporter(monkeypatch, raw) + assert reporter._project_files_source == expected + + def test_returns_none_when_unset_or_garbage(self, monkeypatch): + reporter = self._make_reporter(monkeypatch, None) + assert reporter._project_files_source is None + reporter2 = self._make_reporter(monkeypatch, "Banana") + assert reporter2._project_files_source is None + + def test_post_eval_set_run_payload_carries_source(self, monkeypatch): + from uipath._cli._evals._progress_reporter import StudioWebAgentSnapshot + + reporter = self._make_reporter(monkeypatch, "Local") + spec = reporter._create_eval_set_run_spec( + eval_set_id="test-eval-set", + agent_snapshot=StudioWebAgentSnapshot( + input_schema={"type": "object"}, output_schema={"type": "object"} + ), + no_of_evals=1, + is_coded=False, + ) + assert spec.json["projectFilesSource"] == 1 + + def test_post_eval_run_payload_carries_source(self, monkeypatch): + from uipath.eval.models.evaluation_set import EvaluationItem + + reporter = self._make_reporter(monkeypatch, "Local") + item = EvaluationItem( + id="11111111-1111-1111-1111-111111111111", + name="t", + inputs={}, + evaluation_criterias={}, + ) + spec = reporter._create_eval_run_spec( + eval_item=item, eval_set_run_id="run-1", is_coded=False + ) + assert spec.json["projectFilesSource"] == 1 + + def test_put_eval_run_payload_carries_source(self, monkeypatch): + reporter = self._make_reporter(monkeypatch, "Local") + spec = reporter._update_eval_run_spec( + assertion_runs=[], + evaluator_scores=[], + eval_run_id="run-1", + actual_output={}, + execution_time=1.0, + success=True, + is_coded=False, + ) + assert spec.json["projectFilesSource"] == 1 + + def test_put_coded_eval_run_payload_carries_source(self, monkeypatch): + reporter = self._make_reporter(monkeypatch, "Local") + spec = reporter._update_coded_eval_run_spec( + evaluator_runs=[], + evaluator_scores=[], + eval_run_id="run-1", + actual_output={}, + execution_time=1.0, + success=True, + is_coded=True, + ) + assert spec.json["projectFilesSource"] == 1 + + def test_put_eval_set_run_payload_carries_source(self, monkeypatch): + reporter = self._make_reporter(monkeypatch, "Local") + spec = reporter._update_eval_set_run_spec( + eval_set_run_id="set-run-1", + evaluator_scores={}, + is_coded=False, + success=True, + ) + assert spec.json["projectFilesSource"] == 1 + + def test_get_eval_runs_query_carries_source(self, monkeypatch): + reporter = self._make_reporter(monkeypatch, "Local") + spec = reporter._get_eval_runs_spec( + eval_set_id="set-1", + eval_set_run_id="run-1", + evaluation_id=None, + is_coded=False, + ) + assert spec.params == {"projectFilesSource": 1} + + def test_unset_source_omits_field_from_payloads(self, monkeypatch): + from uipath._cli._evals._progress_reporter import StudioWebAgentSnapshot + + reporter = self._make_reporter(monkeypatch, None) + spec = reporter._create_eval_set_run_spec( + eval_set_id="test-eval-set", + agent_snapshot=StudioWebAgentSnapshot( + input_schema={"type": "object"}, output_schema={"type": "object"} + ), + no_of_evals=1, + is_coded=False, + ) + assert "projectFilesSource" not in spec.json diff --git a/packages/uipath/tests/cli/eval/test_runtime_error_classification.py b/packages/uipath/tests/cli/eval/test_runtime_error_classification.py new file mode 100644 index 000000000..721092b4c --- /dev/null +++ b/packages/uipath/tests/cli/eval/test_runtime_error_classification.py @@ -0,0 +1,171 @@ +"""Tests for classifying eval execution errors as user vs runtime failures.""" + +import uuid +from pathlib import Path +from typing import Any, AsyncGenerator + +from uipath.core.events import EventBus +from uipath.core.tracing import UiPathTraceManager +from uipath.eval.helpers import EvalHelpers +from uipath.eval.runtime import UiPathEvalContext, evaluate +from uipath.eval.runtime.events import EvalRunUpdatedEvent, EvaluationEvents +from uipath.eval.runtime.runtime import _is_user_facing_error +from uipath.runtime import ( + UiPathExecuteOptions, + UiPathRuntimeEvent, + UiPathRuntimeFactorySettings, + UiPathRuntimeProtocol, + UiPathRuntimeResult, + UiPathRuntimeStorageProtocol, + UiPathStreamOptions, +) +from uipath.runtime.errors import ( + UiPathErrorCategory, + UiPathErrorCode, + UiPathRuntimeError, +) +from uipath.runtime.schema import UiPathRuntimeSchema + + +def _make_error(category: UiPathErrorCategory) -> UiPathRuntimeError: + return UiPathRuntimeError( + UiPathErrorCode.FUNCTION_EXECUTION_ERROR, + "Some failure", + "details", + category, + include_traceback=False, + ) + + +def test_user_category_error_is_user_facing(): + assert _is_user_facing_error(_make_error(UiPathErrorCategory.USER)) is True + + +def test_system_category_error_is_not_user_facing(): + assert _is_user_facing_error(_make_error(UiPathErrorCategory.SYSTEM)) is False + + +def test_unknown_category_error_is_not_user_facing(): + assert _is_user_facing_error(_make_error(UiPathErrorCategory.UNKNOWN)) is False + + +def test_plain_exception_is_not_user_facing(): + assert _is_user_facing_error(ValueError("boom")) is False + + +class _FailingRuntime: + """Runtime whose execution always raises the configured error.""" + + def __init__(self, error: Exception): + self._error = error + + async def execute( + self, + input: dict[str, Any] | None = None, + options: UiPathExecuteOptions | None = None, + ) -> UiPathRuntimeResult: + raise self._error + + async def stream( + self, + input: dict[str, Any] | None = None, + options: UiPathStreamOptions | None = None, + ) -> AsyncGenerator[UiPathRuntimeEvent, None]: + raise self._error + yield # unreachable; makes this an async generator + + async def get_schema(self) -> UiPathRuntimeSchema: + return UiPathRuntimeSchema( + filePath="test.py", + uniqueId="test", + type="workflow", + input={"type": "object", "properties": {}}, + output={"type": "object", "properties": {}}, + ) + + async def dispose(self) -> None: + pass + + +class _FailingFactory: + def __init__(self, error: Exception): + self._error = error + + def discover_entrypoints(self) -> list[str]: + return ["test"] + + async def get_storage(self) -> UiPathRuntimeStorageProtocol | None: + return None + + async def get_settings(self) -> UiPathRuntimeFactorySettings | None: + return None + + async def new_runtime( + self, entrypoint: str, runtime_id: str, **kwargs + ) -> UiPathRuntimeProtocol: + return _FailingRuntime(self._error) + + async def dispose(self) -> None: + pass + + +async def _run_failing_eval(error: Exception) -> list[EvalRunUpdatedEvent]: + """Run an eval whose execution raises, returning captured run-updated events.""" + event_bus = EventBus() + trace_manager = UiPathTraceManager() + captured: list[EvalRunUpdatedEvent] = [] + + async def capture(event: EvalRunUpdatedEvent) -> None: + captured.append(event) + + event_bus.subscribe(EvaluationEvents.UPDATE_EVAL_RUN, capture) + + factory = _FailingFactory(error) + eval_set_path = str(Path(__file__).parent / "evals" / "eval-sets" / "default.json") + evaluation_set, _ = EvalHelpers.load_eval_set(eval_set_path) + runtime = await factory.new_runtime("test", "test-runtime-id") + runtime_schema = await runtime.get_schema() + evaluators = await EvalHelpers.load_evaluators( + eval_set_path, evaluation_set, agent_model=None + ) + + context = UiPathEvalContext() + context.execution_id = str(uuid.uuid4()) + context.evaluation_set = evaluation_set + context.runtime_schema = runtime_schema + context.evaluators = evaluators + + await evaluate(factory, trace_manager, context, event_bus) + await event_bus.wait_for_all() + return captured + + +async def test_user_error_execution_is_not_a_runtime_exception(): + """A USER-category failure is reported unwrapped with runtime_exception=False.""" + error = UiPathRuntimeError( + UiPathErrorCode.INPUT_INVALID_JSON, + "Invalid input", + "Input does not match the expected schema", + UiPathErrorCategory.USER, + include_traceback=False, + ) + events = await _run_failing_eval(error) + + failed = [e for e in events if not e.success] + assert failed + details = failed[0].exception_details + assert details is not None + assert details.exception is error + assert details.runtime_exception is False + + +async def test_unexpected_error_execution_is_a_runtime_exception(): + """A non-user failure in the execution path is flagged as a runtime exception.""" + events = await _run_failing_eval(RuntimeError("infrastructure broke")) + + failed = [e for e in events if not e.success] + assert failed + details = failed[0].exception_details + assert details is not None + assert isinstance(details.exception, RuntimeError) + assert details.runtime_exception is True diff --git a/packages/uipath/tests/cli/evaluators/test_json_similarity_evaluator.py b/packages/uipath/tests/cli/evaluators/test_json_similarity_evaluator.py index 0b74ae07c..1ce823f22 100644 --- a/packages/uipath/tests/cli/evaluators/test_json_similarity_evaluator.py +++ b/packages/uipath/tests/cli/evaluators/test_json_similarity_evaluator.py @@ -11,9 +11,9 @@ from uipath.eval.evaluators import LegacyJsonSimilarityEvaluator from uipath.eval.evaluators.base_legacy_evaluator import LegacyEvaluationCriteria from uipath.eval.models.models import ( - AgentExecution, LegacyEvaluatorCategory, LegacyEvaluatorType, + WorkloadExecution, ) @@ -65,10 +65,10 @@ async def test_json_similarity_exact_score_1(self) -> None: """ result = await evaluator.evaluate( - AgentExecution( + WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output=json.loads(actual_json), + workload_trace=[], + workload_output=json.loads(actual_json), ), evaluation_criteria=LegacyEvaluationCriteria( expected_output=json.loads(expected_json), @@ -101,10 +101,10 @@ async def test_json_similarity_exact_score_2(self) -> None: """ result = await evaluator.evaluate( - AgentExecution( + WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output=json.loads(actual_json), + workload_trace=[], + workload_output=json.loads(actual_json), ), evaluation_criteria=LegacyEvaluationCriteria( expected_output=json.loads(expected_json), @@ -135,10 +135,10 @@ async def test_json_similarity_exact_score_3(self) -> None: """ result = await evaluator.evaluate( - AgentExecution( + WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output=json.loads(actual_json), + workload_trace=[], + workload_output=json.loads(actual_json), ), evaluation_criteria=LegacyEvaluationCriteria( expected_output=json.loads(expected_json), @@ -229,10 +229,10 @@ async def test_json_similarity_exact_score_4(self) -> None: """ result = await evaluator.evaluate( - AgentExecution( + WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output=json.loads(actual_json), + workload_trace=[], + workload_output=json.loads(actual_json), ), evaluation_criteria=LegacyEvaluationCriteria( expected_output=json.loads(expected_json), diff --git a/packages/uipath/tests/cli/evaluators/test_legacy_context_precision_evaluator.py b/packages/uipath/tests/cli/evaluators/test_legacy_context_precision_evaluator.py index 19723eb25..cd1c28738 100644 --- a/packages/uipath/tests/cli/evaluators/test_legacy_context_precision_evaluator.py +++ b/packages/uipath/tests/cli/evaluators/test_legacy_context_precision_evaluator.py @@ -13,9 +13,9 @@ from uipath.eval.evaluators import LegacyContextPrecisionEvaluator from uipath.eval.evaluators.base_legacy_evaluator import LegacyEvaluationCriteria from uipath.eval.models.models import ( - AgentExecution, LegacyEvaluatorCategory, LegacyEvaluatorType, + WorkloadExecution, ) @@ -205,14 +205,14 @@ async def test_evaluation_with_no_context_groundings( evaluator = evaluator_with_mocked_llm # Create empty agent execution (no spans) - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output="", + workload_trace=[], + workload_output="", ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output="", expected_agent_behavior="", diff --git a/packages/uipath/tests/cli/evaluators/test_legacy_csv_exact_match_evaluator.py b/packages/uipath/tests/cli/evaluators/test_legacy_csv_exact_match_evaluator.py index 1fd2f4f9f..45cdfaefe 100644 --- a/packages/uipath/tests/cli/evaluators/test_legacy_csv_exact_match_evaluator.py +++ b/packages/uipath/tests/cli/evaluators/test_legacy_csv_exact_match_evaluator.py @@ -19,9 +19,9 @@ ) from uipath.eval.evaluators.output_evaluator import LineByLineEvaluationDetails from uipath.eval.models.models import ( - AgentExecution, LegacyEvaluatorCategory, LegacyEvaluatorType, + WorkloadExecution, ) @@ -64,14 +64,14 @@ class TestLegacyCSVExactMatchEvaluator: async def test_single_column_match(self, evaluator_single_column) -> None: """Test exact match with single column.""" csv_content = "Name,Age,City\nJohn,25,Paris" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output=csv_content, + workload_trace=[], + workload_output=csv_content, ) result = await evaluator_single_column.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"content": csv_content}, expected_agent_behavior="", @@ -84,14 +84,14 @@ async def test_single_column_match(self, evaluator_single_column) -> None: async def test_multiple_columns_match(self, evaluator_multiple_columns) -> None: """Test exact match with multiple columns.""" csv_content = "Name,Age,City\nJohn,25,Paris" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output=csv_content, + workload_trace=[], + workload_output=csv_content, ) result = await evaluator_multiple_columns.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"content": csv_content}, expected_agent_behavior="", @@ -106,14 +106,14 @@ async def test_column_value_differs(self, evaluator_single_column) -> None: actual_csv = "Name,Age,City\nJohn,25,Paris" expected_csv = "Name,Age,City\nJane,25,Paris" # Different name - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output=actual_csv, + workload_trace=[], + workload_output=actual_csv, ) result = await evaluator_single_column.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"content": expected_csv}, expected_agent_behavior="", @@ -131,14 +131,14 @@ async def test_case_insensitive_column_names(self) -> None: ) csv_content = "name,AGE,CiTy\nJohn,25,Paris" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output=csv_content, + workload_trace=[], + workload_output=csv_content, ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"content": csv_content}, expected_agent_behavior="", @@ -153,14 +153,14 @@ async def test_json_element_with_content_property( ) -> None: """Test handling dict with 'content' property.""" csv_content = "Name,Age\nJohn,25" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output={"content": csv_content}, + workload_trace=[], + workload_output={"content": csv_content}, ) result = await evaluator_single_column.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"content": csv_content}, expected_agent_behavior="", @@ -193,14 +193,14 @@ async def test_column_not_found_returns_error( **_make_base_params(target_sub_output_key="NonExistentColumn") ) - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output=csv_content, + workload_trace=[], + workload_output=csv_content, ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"content": csv_content}, expected_agent_behavior="", @@ -214,14 +214,14 @@ async def test_column_not_found_returns_error( @pytest.mark.asyncio async def test_empty_csv_returns_error(self, evaluator_single_column) -> None: """Test that empty CSV returns error result.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output="", + workload_trace=[], + workload_output="", ) result = await evaluator_single_column.validate_and_evaluate_criteria( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"content": "Name,Age\nJohn,25"}, expected_agent_behavior="", @@ -244,14 +244,14 @@ async def test_complex_csv_with_quotes_and_commas( csv_content = ( 'Name,Description,Status\n"John, Jr.","Software Engineer, Senior","Active"' ) - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output=csv_content, + workload_trace=[], + workload_output=csv_content, ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"content": csv_content}, expected_agent_behavior="", @@ -269,14 +269,14 @@ async def test_empty_values(self) -> None: ) csv_content = "Name,Age,City\nJohn,,Paris" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output=csv_content, + workload_trace=[], + workload_output=csv_content, ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"content": csv_content}, expected_agent_behavior="", @@ -289,14 +289,14 @@ async def test_empty_values(self) -> None: async def test_no_data_rows_returns_error(self, evaluator_single_column) -> None: """Test that CSV with headers but no data rows returns error result.""" csv_content = "Name,Age" # Headers only, no data - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output=csv_content, + workload_trace=[], + workload_output=csv_content, ) result = await evaluator_single_column.validate_and_evaluate_criteria( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"content": csv_content}, expected_agent_behavior="", @@ -315,14 +315,14 @@ async def test_whitespace_only_target_columns_returns_error(self) -> None: ) csv_content = "Name,Age\nJohn,25" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output=csv_content, + workload_trace=[], + workload_output=csv_content, ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"content": csv_content}, expected_agent_behavior="", @@ -339,15 +339,15 @@ async def test_expected_output_without_content_property_returns_error( ) -> None: """Test that expected output without 'content' property returns error result.""" csv_content = "Name,Age\nJohn,25" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output=csv_content, + workload_trace=[], + workload_output=csv_content, ) # Expected output as empty string result = await evaluator_single_column.validate_and_evaluate_criteria( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output="", expected_agent_behavior="", @@ -371,14 +371,14 @@ async def test_actual_missing_column_in_comparison(self) -> None: actual_csv = "Name\nJohn" # Missing Age column expected_csv = "Name,Age\nJohn,25" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output=actual_csv, + workload_trace=[], + workload_output=actual_csv, ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"content": expected_csv}, expected_agent_behavior="", @@ -395,14 +395,14 @@ async def test_whitespace_trimming_in_values(self, evaluator_single_column) -> N actual_csv = "Name,Age\n John ,25" expected_csv = "Name,Age\nJohn,25" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output=actual_csv, + workload_trace=[], + workload_output=actual_csv, ) result = await evaluator_single_column.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"content": expected_csv}, expected_agent_behavior="", @@ -419,14 +419,14 @@ async def test_case_sensitive_value_comparison( actual_csv = "Name,Age\nJohn,25" expected_csv = "Name,Age\njohn,25" # Different case - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output=actual_csv, + workload_trace=[], + workload_output=actual_csv, ) result = await evaluator_single_column.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"content": expected_csv}, expected_agent_behavior="", @@ -441,14 +441,14 @@ async def test_extra_columns_are_ignored(self, evaluator_single_column) -> None: actual_csv = "Name,Age,City,Country\nJohn,25,Paris,France" expected_csv = "Name,Age,City\nJohn,30,London" # Different Age and City, but we only check Name - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output=actual_csv, + workload_trace=[], + workload_output=actual_csv, ) result = await evaluator_single_column.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"content": expected_csv}, expected_agent_behavior="", @@ -472,14 +472,14 @@ async def test_line_by_line_all_match(): ) csv_content = "Name,Age\nJohn,25\nJane,30\nBob,35" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"input": "test"}, - agent_output=csv_content, - agent_trace=[], + workload_output=csv_content, + workload_trace=[], ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"content": csv_content}, expected_agent_behavior="", @@ -509,14 +509,14 @@ async def test_line_by_line_partial_match(): actual_csv = "Name,Age\nJohn,25\nDifferent,30\nBob,35" expected_csv = "Name,Age\nJohn,25\nJane,30\nBob,35" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"input": "test"}, - agent_output=actual_csv, - agent_trace=[], + workload_output=actual_csv, + workload_trace=[], ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"content": expected_csv}, expected_agent_behavior="", @@ -548,14 +548,14 @@ async def test_line_by_line_multiple_columns(): ) csv_content = "Name,Age,City\nJohn,25,Paris\nJane,30,London" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"input": "test"}, - agent_output=csv_content, - agent_trace=[], + workload_output=csv_content, + workload_trace=[], ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"content": csv_content}, expected_agent_behavior="", @@ -582,14 +582,14 @@ async def test_line_by_line_unequal_line_counts(): actual_csv = "Name,Age\nJohn,25\nJane,30" # 2 data rows expected_csv = "Name,Age\nJohn,25\nJane,30\nBob,35" # 3 data rows - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"input": "test"}, - agent_output=actual_csv, - agent_trace=[], + workload_output=actual_csv, + workload_trace=[], ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"content": expected_csv}, expected_agent_behavior="", @@ -621,15 +621,15 @@ async def test_line_by_line_with_job_attachment(): "uipath.eval.evaluators.base_legacy_evaluator.download_attachment_as_string", return_value=csv_content, ): - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"input": "test"}, # Simulate job attachment URI - agent_output="urn:uipath:cas:file:orchestrator:12345678-1234-1234-1234-123456789abc", - agent_trace=[], + workload_output="urn:uipath:cas:file:orchestrator:12345678-1234-1234-1234-123456789abc", + workload_trace=[], ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"content": csv_content}, expected_agent_behavior="", diff --git a/packages/uipath/tests/cli/evaluators/test_legacy_exact_match_evaluator.py b/packages/uipath/tests/cli/evaluators/test_legacy_exact_match_evaluator.py index b5b5bfd4f..cb1d5d9df 100644 --- a/packages/uipath/tests/cli/evaluators/test_legacy_exact_match_evaluator.py +++ b/packages/uipath/tests/cli/evaluators/test_legacy_exact_match_evaluator.py @@ -13,9 +13,9 @@ from uipath.eval.evaluators.base_legacy_evaluator import LegacyEvaluationCriteria from uipath.eval.evaluators.output_evaluator import LineByLineEvaluationDetails from uipath.eval.models.models import ( - AgentExecution, LegacyEvaluatorCategory, LegacyEvaluatorType, + WorkloadExecution, ) @@ -55,14 +55,14 @@ class TestLegacyExactMatchEvaluator: @pytest.mark.asyncio async def test_exact_match_same_strings(self, evaluator) -> None: """Test exact match with identical string outputs.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output="Hello World", + workload_trace=[], + workload_output="Hello World", ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output="Hello World", expected_agent_behavior="", @@ -75,14 +75,14 @@ async def test_exact_match_same_strings(self, evaluator) -> None: @pytest.mark.asyncio async def test_exact_match_different_strings(self, evaluator) -> None: """Test exact match with different string outputs.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output="Hello World", + workload_trace=[], + workload_output="Hello World", ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output="Goodbye World", expected_agent_behavior="", @@ -95,14 +95,14 @@ async def test_exact_match_different_strings(self, evaluator) -> None: async def test_exact_match_identical_dicts(self, evaluator) -> None: """Test exact match with identical dictionaries.""" output = {"name": "John", "age": 30} - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output=output, + workload_trace=[], + workload_output=output, ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output=output, expected_agent_behavior="", @@ -114,14 +114,14 @@ async def test_exact_match_identical_dicts(self, evaluator) -> None: @pytest.mark.asyncio async def test_exact_match_different_dicts(self, evaluator) -> None: """Test exact match with different dictionaries.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output={"name": "John", "age": 30}, + workload_trace=[], + workload_output={"name": "John", "age": 30}, ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"name": "Jane", "age": 25}, expected_agent_behavior="", @@ -133,14 +133,14 @@ async def test_exact_match_different_dicts(self, evaluator) -> None: @pytest.mark.asyncio async def test_exact_match_dict_key_order_doesnt_matter(self, evaluator) -> None: """Test that canonical JSON normalization handles key order.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output={"a": 1, "b": 2}, + workload_trace=[], + workload_output={"a": 1, "b": 2}, ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"b": 2, "a": 1}, expected_agent_behavior="", @@ -154,14 +154,14 @@ async def test_exact_match_number_normalization_int_to_float( self, evaluator ) -> None: """Test that integers are normalized to floats for comparison.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output={"value": 42}, + workload_trace=[], + workload_output={"value": 42}, ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"value": 42.0}, expected_agent_behavior="", @@ -173,14 +173,14 @@ async def test_exact_match_number_normalization_int_to_float( @pytest.mark.asyncio async def test_exact_match_number_normalization_in_list(self, evaluator) -> None: """Test number normalization in lists.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output={"values": [1, 2, 3]}, + workload_trace=[], + workload_output={"values": [1, 2, 3]}, ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"values": [1.0, 2.0, 3.0]}, expected_agent_behavior="", @@ -192,14 +192,14 @@ async def test_exact_match_number_normalization_in_list(self, evaluator) -> None @pytest.mark.asyncio async def test_exact_match_booleans_preserved(self, evaluator) -> None: """Test that booleans are not converted to numbers.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output={"active": True, "deleted": False}, + workload_trace=[], + workload_output={"active": True, "deleted": False}, ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"active": True, "deleted": False}, expected_agent_behavior="", @@ -220,14 +220,14 @@ async def test_exact_match_nested_structures(self, evaluator) -> None: "metadata": {"version": 1.0}, } - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output=output, + workload_trace=[], + workload_output=output, ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output=output, expected_agent_behavior="", @@ -241,14 +241,14 @@ async def test_exact_match_with_target_key_both_have_key( self, evaluator_with_target_key ) -> None: """Test target_output_key extraction when both outputs have the key.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output={"result": {"status": "success"}, "other": "ignore"}, + workload_trace=[], + workload_output={"result": {"status": "success"}, "other": "ignore"}, ) result = await evaluator_with_target_key.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"result": {"status": "success"}, "other": "different"}, expected_agent_behavior="", @@ -263,14 +263,14 @@ async def test_exact_match_with_target_key_missing_in_both( self, evaluator_with_target_key ) -> None: """Test target_output_key when key is missing in both outputs.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output={"other": "data"}, + workload_trace=[], + workload_output={"other": "data"}, ) result = await evaluator_with_target_key.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"other": "different"}, expected_agent_behavior="", @@ -285,14 +285,14 @@ async def test_exact_match_with_target_key_missing_in_actual( self, evaluator_with_target_key ) -> None: """Test target_output_key when key is missing in actual output.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output={"other": "data"}, + workload_trace=[], + workload_output={"other": "data"}, ) result = await evaluator_with_target_key.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"result": {"status": "success"}}, expected_agent_behavior="", @@ -307,14 +307,14 @@ async def test_exact_match_with_target_key_missing_in_expected( self, evaluator_with_target_key ) -> None: """Test target_output_key when key is missing in expected output.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output={"result": {"status": "success"}}, + workload_trace=[], + workload_output={"result": {"status": "success"}}, ) result = await evaluator_with_target_key.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"other": "data"}, expected_agent_behavior="", @@ -327,14 +327,14 @@ async def test_exact_match_with_target_key_missing_in_expected( @pytest.mark.asyncio async def test_exact_match_with_wildcard_target_key(self, evaluator) -> None: """Test that wildcard target_output_key compares full outputs.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output={"data": "value", "extra": "field"}, + workload_trace=[], + workload_output={"data": "value", "extra": "field"}, ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"data": "value", "extra": "field"}, expected_agent_behavior="", @@ -348,14 +348,14 @@ async def test_exact_match_with_target_key_non_dict_inputs( self, evaluator_with_target_key ) -> None: """Test target_output_key with non-dict inputs (should compare as-is).""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output="string_value", + workload_trace=[], + workload_output="string_value", ) result = await evaluator_with_target_key.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output="other_string", expected_agent_behavior="", @@ -367,14 +367,14 @@ async def test_exact_match_with_target_key_non_dict_inputs( @pytest.mark.asyncio async def test_exact_match_null_values(self, evaluator) -> None: """Test exact match with None values.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output={"value": None}, + workload_trace=[], + workload_output={"value": None}, ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"value": None}, expected_agent_behavior="", @@ -386,14 +386,14 @@ async def test_exact_match_null_values(self, evaluator) -> None: @pytest.mark.asyncio async def test_exact_match_empty_dict(self, evaluator) -> None: """Test exact match with empty dictionaries.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output={}, + workload_trace=[], + workload_output={}, ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={}, expected_agent_behavior="", @@ -405,14 +405,14 @@ async def test_exact_match_empty_dict(self, evaluator) -> None: @pytest.mark.asyncio async def test_exact_match_empty_string(self, evaluator) -> None: """Test exact match with empty strings.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output="", + workload_trace=[], + workload_output="", ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output="", expected_agent_behavior="", @@ -425,14 +425,14 @@ async def test_exact_match_empty_string(self, evaluator) -> None: async def test_exact_match_unicode_characters(self, evaluator) -> None: """Test exact match with unicode characters.""" output = {"greeting": "你好世界", "emoji": "🎉"} - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output=output, + workload_trace=[], + workload_output=output, ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output=output, expected_agent_behavior="", @@ -444,14 +444,14 @@ async def test_exact_match_unicode_characters(self, evaluator) -> None: @pytest.mark.asyncio async def test_exact_match_whitespace_matters(self, evaluator) -> None: """Test that whitespace differences are detected.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output="Hello World", + workload_trace=[], + workload_output="Hello World", ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output="Hello World", expected_agent_behavior="", @@ -463,14 +463,14 @@ async def test_exact_match_whitespace_matters(self, evaluator) -> None: @pytest.mark.asyncio async def test_exact_match_case_sensitivity(self, evaluator) -> None: """Test that string comparison is case sensitive.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output="Hello", + workload_trace=[], + workload_output="Hello", ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output="hello", expected_agent_behavior="", @@ -482,14 +482,14 @@ async def test_exact_match_case_sensitivity(self, evaluator) -> None: @pytest.mark.asyncio async def test_exact_match_large_numbers(self, evaluator) -> None: """Test exact match with large numbers.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output={"value": 999999999999999}, + workload_trace=[], + workload_output={"value": 999999999999999}, ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"value": 999999999999999.0}, expected_agent_behavior="", @@ -501,14 +501,14 @@ async def test_exact_match_large_numbers(self, evaluator) -> None: @pytest.mark.asyncio async def test_exact_match_floating_point_precision(self, evaluator) -> None: """Test exact match with floating point numbers.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output={"pi": 3.14159}, + workload_trace=[], + workload_output={"pi": 3.14159}, ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"pi": 3.14159}, expected_agent_behavior="", @@ -520,14 +520,14 @@ async def test_exact_match_floating_point_precision(self, evaluator) -> None: @pytest.mark.asyncio async def test_exact_match_float_vs_int_zero(self, evaluator) -> None: """Test that 0 and 0.0 are considered equal.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output={"value": 0}, + workload_trace=[], + workload_output={"value": 0}, ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"value": 0.0}, expected_agent_behavior="", @@ -541,14 +541,14 @@ async def test_exact_match_with_target_key_different_values( self, evaluator_with_target_key ) -> None: """Test target_output_key with different values in target key.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output={"result": {"status": "success"}, "other": "ignore"}, + workload_trace=[], + workload_output={"result": {"status": "success"}, "other": "ignore"}, ) result = await evaluator_with_target_key.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"result": {"status": "failed"}, "other": "ignore"}, expected_agent_behavior="", @@ -561,10 +561,10 @@ async def test_exact_match_with_target_key_different_values( async def test_canonical_json_normalization(self, evaluator) -> None: """Test that canonical JSON normalization works correctly.""" # Create complex nested structure with mixed types - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output={ + workload_trace=[], + workload_output={ "z_key": 1, "a_key": [3, 2, 1], "m_key": {"nested": 42}, @@ -572,7 +572,7 @@ async def test_canonical_json_normalization(self, evaluator) -> None: ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={ "a_key": [3, 2, 1], @@ -597,14 +597,14 @@ async def test_line_by_line_all_match(): lineDelimiter="\n", ) - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"input": "test"}, - agent_output="Line 1\nLine 2\nLine 3", - agent_trace=[], + workload_output="Line 1\nLine 2\nLine 3", + workload_trace=[], ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output="Line 1\nLine 2\nLine 3", expected_agent_behavior="", @@ -631,14 +631,14 @@ async def test_line_by_line_partial_match(): lineDelimiter="\n", ) - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"input": "test"}, - agent_output="Line 1\nDifferent\nLine 3", - agent_trace=[], + workload_output="Line 1\nDifferent\nLine 3", + workload_trace=[], ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output="Line 1\nLine 2\nLine 3", expected_agent_behavior="", @@ -669,14 +669,14 @@ async def test_line_by_line_with_target_output_key(): lineDelimiter="\n", ) - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"input": "test"}, - agent_output={"result": "Line 1\nLine 2\nLine 3"}, - agent_trace=[], + workload_output={"result": "Line 1\nLine 2\nLine 3"}, + workload_trace=[], ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output={"result": "Line 1\nLine 2\nLine 3"}, expected_agent_behavior="", @@ -701,14 +701,14 @@ async def test_line_by_line_custom_delimiter(): lineDelimiter="|", ) - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"input": "test"}, - agent_output="Item1|Item2|Item3", - agent_trace=[], + workload_output="Item1|Item2|Item3", + workload_trace=[], ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output="Item1|Item2|Item3", expected_agent_behavior="", @@ -734,14 +734,14 @@ async def test_line_by_line_unequal_line_counts(): lineDelimiter="\n", ) - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"input": "test"}, - agent_output="Line 1\nLine 2", - agent_trace=[], + workload_output="Line 1\nLine 2", + workload_trace=[], ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output="Line 1\nLine 2\nLine 3", expected_agent_behavior="", diff --git a/packages/uipath/tests/cli/evaluators/test_legacy_faithfulness_evaluator.py b/packages/uipath/tests/cli/evaluators/test_legacy_faithfulness_evaluator.py index 68d8b407b..4e609575b 100644 --- a/packages/uipath/tests/cli/evaluators/test_legacy_faithfulness_evaluator.py +++ b/packages/uipath/tests/cli/evaluators/test_legacy_faithfulness_evaluator.py @@ -13,9 +13,9 @@ from uipath.eval.evaluators import LegacyFaithfulnessEvaluator from uipath.eval.evaluators.base_legacy_evaluator import LegacyEvaluationCriteria from uipath.eval.models.models import ( - AgentExecution, LegacyEvaluatorCategory, LegacyEvaluatorType, + WorkloadExecution, ) @@ -142,7 +142,7 @@ async def test_select_verifiable_sentences(self, evaluator_with_mocked_llm) -> N """Test Stage 1: Selection of verifiable sentences.""" evaluator = evaluator_with_mocked_llm - agent_output = ( + workload_output = ( "The capital of France is Paris. Do you agree? This is important." ) @@ -153,7 +153,7 @@ async def test_select_verifiable_sentences(self, evaluator_with_mocked_llm) -> N ) as mock_llm: mock_llm.return_value = mock_response - sentences = await evaluator._select_verifiable_sentences(agent_output) + sentences = await evaluator._select_verifiable_sentences(workload_output) assert len(sentences) == 1 assert "capital of France" in sentences[0] @@ -355,14 +355,14 @@ async def test_full_evaluation_with_no_agent_output( """Test evaluation when no agent output is provided.""" evaluator = evaluator_with_mocked_llm - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output="", + workload_trace=[], + workload_output="", ) result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output="", expected_agent_behavior="", @@ -370,7 +370,7 @@ async def test_full_evaluation_with_no_agent_output( ) assert result.score == 0.0 - assert "no agent output" in result.details.lower() + assert "no workload output" in result.details.lower() @pytest.mark.asyncio async def test_full_evaluation_with_no_context_sources( @@ -388,15 +388,15 @@ class NoOutputSpan: } ) - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output="The sky is blue.", + workload_trace=[], + workload_output="The sky is blue.", ) with patch.object(evaluator, "_extract_context_sources", return_value=[]): result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output="The sky is blue.", expected_agent_behavior="", @@ -413,10 +413,10 @@ async def test_full_evaluation_with_no_verifiable_claims( """Test evaluation when no verifiable claims are found.""" evaluator = evaluator_with_mocked_llm - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output="Just a greeting.", + workload_trace=[], + workload_output="Just a greeting.", ) with ( @@ -432,7 +432,7 @@ async def test_full_evaluation_with_no_verifiable_claims( mock_claims.return_value = [] result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output="Just a greeting.", expected_agent_behavior="", @@ -449,10 +449,10 @@ async def test_full_evaluation_with_grounded_claims( """Test full evaluation flow with grounded claims.""" evaluator = evaluator_with_mocked_llm - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output="Paris is in France.", + workload_trace=[], + workload_output="Paris is in France.", ) # Mock the extraction and evaluation steps @@ -488,7 +488,7 @@ async def test_full_evaluation_with_grounded_claims( ] result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output="Paris is in France.", expected_agent_behavior="", @@ -506,10 +506,10 @@ async def test_full_evaluation_with_mixed_claims( """Test full evaluation with both grounded and ungrounded claims.""" evaluator = evaluator_with_mocked_llm - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_trace=[], - agent_output="Paris is in France. The sky is green.", + workload_trace=[], + workload_output="Paris is in France. The sky is green.", ) # Mock the extraction and evaluation steps @@ -553,7 +553,7 @@ async def test_full_evaluation_with_mixed_claims( ] result = await evaluator.evaluate( - agent_execution, + workload_execution, evaluation_criteria=LegacyEvaluationCriteria( expected_output="Paris is in France. The sky is green.", expected_agent_behavior="", diff --git a/packages/uipath/tests/cli/integration/test_list_models_commands.py b/packages/uipath/tests/cli/integration/test_list_models_commands.py new file mode 100644 index 000000000..c7cb9f042 --- /dev/null +++ b/packages/uipath/tests/cli/integration/test_list_models_commands.py @@ -0,0 +1,233 @@ +"""Integration tests for the `uipath list-models` CLI command. + +The command renders a rich table grouped by vendor (one column per vendor) +for human terminal use, and falls through to the shared `format_output` +pipeline for `--format json|csv` and `--output `. +""" + +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from click.testing import CliRunner + +from uipath._cli import cli +from uipath.platform.agenthub import LlmModel + + +@pytest.fixture +def runner(): + """Provide a Click CLI test runner.""" + return CliRunner() + + +@pytest.fixture +def mock_client(): + """Provide a mocked UiPath client with an async agenthub service.""" + with patch("uipath.platform._uipath.UiPath") as mock: + client_instance = MagicMock() + mock.return_value = client_instance + + client_instance.agenthub = MagicMock() + client_instance.agenthub.get_available_llm_models_async = AsyncMock() + + yield client_instance + + +def _make_models() -> list[LlmModel]: + """Build a small list of LlmModel instances spanning multiple vendors.""" + return [ + LlmModel(model_name="gpt-4o-mini", vendor="OpenAi"), + LlmModel(model_name="gpt-4.1", vendor="OpenAi"), + LlmModel(model_name="claude-sonnet-4-5", vendor="Anthropic"), + LlmModel(model_name="gemini-2.5-flash", vendor="VertexAi"), + ] + + +class TestRichTable: + def test_renders_each_model_and_vendor(self, runner, mock_client, mock_env_vars): + """All models and vendor headers appear in the rendered table.""" + mock_client.agenthub.get_available_llm_models_async.return_value = ( + _make_models() + ) + + result = runner.invoke(cli, ["list-models"]) + + assert result.exit_code == 0 + for model in _make_models(): + assert model.model_name in result.output + assert (model.vendor or "") in result.output + mock_client.agenthub.get_available_llm_models_async.assert_awaited_once() + + def test_table_title(self, runner, mock_client, mock_env_vars): + """The rich table renders its title for orientation.""" + mock_client.agenthub.get_available_llm_models_async.return_value = ( + _make_models() + ) + + result = runner.invoke(cli, ["list-models"]) + + assert result.exit_code == 0 + assert "Available LLM Models" in result.output + + def test_missing_vendor_grouped_under_unknown( + self, runner, mock_client, mock_env_vars + ): + """A model with no vendor lands in an 'Unknown' column.""" + mock_client.agenthub.get_available_llm_models_async.return_value = [ + LlmModel(model_name="custom-model", vendor=None), + ] + + result = runner.invoke(cli, ["list-models"]) + + assert result.exit_code == 0 + assert "custom-model" in result.output + assert "Unknown" in result.output + + def test_empty(self, runner, mock_client, mock_env_vars): + """An empty model list renders the title without rows or errors.""" + mock_client.agenthub.get_available_llm_models_async.return_value = [] + + result = runner.invoke(cli, ["list-models"]) + + assert result.exit_code == 0 + assert "Available LLM Models" in result.output + + +class TestMachineReadableFormats: + def test_json_format(self, runner, mock_client, mock_env_vars): + """--format json bypasses the rich table and emits parseable JSON.""" + mock_client.agenthub.get_available_llm_models_async.return_value = ( + _make_models() + ) + + result = runner.invoke(cli, ["list-models", "--format", "json"]) + + assert result.exit_code == 0 + payload = json.loads(result.output) + assert isinstance(payload, list) + assert {m["model_name"] for m in payload} == { + "gpt-4o-mini", + "gpt-4.1", + "claude-sonnet-4-5", + "gemini-2.5-flash", + } + + def test_csv_format(self, runner, mock_client, mock_env_vars): + """--format csv emits a header row and one row per model.""" + mock_client.agenthub.get_available_llm_models_async.return_value = ( + _make_models() + ) + + result = runner.invoke(cli, ["list-models", "--format", "csv"]) + + assert result.exit_code == 0 + lines = [line for line in result.output.splitlines() if line.strip()] + assert "model_name" in lines[0] + assert "vendor" in lines[0] + assert any("gpt-4o-mini" in line for line in lines[1:]) + + def test_global_json_flag(self, runner, mock_client, mock_env_vars): + """The cli-group --format json is honored too.""" + mock_client.agenthub.get_available_llm_models_async.return_value = ( + _make_models() + ) + + result = runner.invoke(cli, ["--format", "json", "list-models"]) + + assert result.exit_code == 0 + payload = json.loads(result.output) + assert len(payload) == 4 + + def test_output_writes_through_plain_formatter( + self, runner, mock_client, mock_env_vars, tmp_path + ): + """--output writes through format_output (not the rich path).""" + mock_client.agenthub.get_available_llm_models_async.return_value = ( + _make_models() + ) + out_file = tmp_path / "models.json" + + result = runner.invoke( + cli, + ["list-models", "--format", "json", "--output", str(out_file)], + ) + + assert result.exit_code == 0 + assert out_file.exists() + payload = json.loads(out_file.read_text(encoding="utf-8")) + assert {m["model_name"] for m in payload} == { + "gpt-4o-mini", + "gpt-4.1", + "claude-sonnet-4-5", + "gemini-2.5-flash", + } + + def test_output_file_alias(self, runner, mock_client, mock_env_vars, tmp_path): + """`--output-file` works as an alias for `--output` (matches `run`).""" + mock_client.agenthub.get_available_llm_models_async.return_value = ( + _make_models() + ) + out_file = tmp_path / "models.json" + + result = runner.invoke( + cli, + ["list-models", "--format", "json", "--output-file", str(out_file)], + ) + + assert result.exit_code == 0 + assert out_file.exists() + payload = json.loads(out_file.read_text(encoding="utf-8")) + assert len(payload) == 4 + + +class TestErrorPaths: + def test_service_error(self, runner, mock_client, mock_env_vars): + """Exceptions from the service are surfaced as click errors.""" + mock_client.agenthub.get_available_llm_models_async.side_effect = RuntimeError( + "boom" + ) + + result = runner.invoke(cli, ["list-models"]) + + assert result.exit_code != 0 + assert "boom" in result.output + + def test_missing_url(self, runner, monkeypatch): + """Missing UIPATH_URL surfaces an auth-configuration error.""" + monkeypatch.delenv("UIPATH_URL", raising=False) + monkeypatch.setenv("UIPATH_ACCESS_TOKEN", "mock_token") + + result = runner.invoke(cli, ["list-models"]) + + assert result.exit_code != 0 + assert "UIPATH_URL not configured" in result.output + + def test_missing_token(self, runner, monkeypatch): + """Missing UIPATH_ACCESS_TOKEN surfaces an auth-configuration error.""" + monkeypatch.setenv("UIPATH_URL", "https://cloud.uipath.com/org/tenant") + monkeypatch.delenv("UIPATH_ACCESS_TOKEN", raising=False) + + result = runner.invoke(cli, ["list-models"]) + + assert result.exit_code != 0 + assert "Authentication required" in result.output + + +class TestRegistration: + def test_help_text(self, runner): + """--help surfaces the command description and options.""" + result = runner.invoke(cli, ["list-models", "--help"]) + + assert result.exit_code == 0 + assert "List available LLM models" in result.output + assert "--format" in result.output + assert "--output" in result.output + assert "--output-file" in result.output + + def test_registered_in_cli(self, runner): + """The command is wired into the top-level CLI group.""" + result = runner.invoke(cli, ["--help"]) + + assert result.exit_code == 0 + assert "list-models" in result.output diff --git a/packages/uipath/tests/cli/integration/test_trace_shutdown_server.py b/packages/uipath/tests/cli/integration/test_trace_shutdown_server.py new file mode 100644 index 000000000..754c2fe10 --- /dev/null +++ b/packages/uipath/tests/cli/integration/test_trace_shutdown_server.py @@ -0,0 +1,155 @@ +import asyncio +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Generator, cast + +import pytest + +from tests.cli.utils.server import ( + get_free_port, + start_cli_server_thread, + start_job_with_env, +) + + +class TraceServer(ThreadingHTTPServer): + posts: list[bytes] + + +class TraceHandler(BaseHTTPRequestHandler): + def do_POST(self) -> None: + length = int(self.headers.get("content-length", "0")) + body = self.rfile.read(length) + cast(TraceServer, self.server).posts.append(body) + self.send_response(200) + self.end_headers() + self.wfile.write(b"OK") + + def log_message(self, format: str, *args: object) -> None: + return + + +@pytest.fixture +def trace_stub() -> Generator[tuple[TraceServer, str], None, None]: + port = get_free_port() + server = TraceServer(("127.0.0.1", port), TraceHandler) + server.posts = [] + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + + try: + yield server, f"http://127.0.0.1:{port}" + finally: + server.shutdown() + server.server_close() + + +def write_project(project: Path) -> None: + (project / "entrypoint.py").write_text( + """from dataclasses import dataclass +from uipath.tracing import traced + + +@dataclass +class Input: + message: str + + +@dataclass +class Output: + message: str + + +@traced(name="actual-agent-span") +def main(input: Input) -> Output: + return Output(message=f"ok: {input.message}") +""", + encoding="utf-8", + ) + (project / "uipath.json").write_text( + json.dumps({"agents": {"main": "entrypoint.py:main"}}, indent=2), + encoding="utf-8", + ) + + +async def run_traced_job( + port: int, + project: Path, + trace_url: str, + job_key: str, + command: str, + extra_args: list[str], +) -> tuple[dict[str, Any], Path]: + input_file = project / f"{job_key}-input.json" + output_file = project / f"{job_key}-output.json" + input_file.write_text(json.dumps({"message": job_key}), encoding="utf-8") + + response = await start_job_with_env( + port, + job_key, + command, + [ + "main", + "--input-file", + str(input_file), + "--output-file", + str(output_file), + *extra_args, + ], + { + "UIPATH_ACCESS_TOKEN": "fake-token", + "UIPATH_JOB_KEY": job_key, + "UIPATH_ORGANIZATION_ID": "org-123", + "UIPATH_TENANT_ID": "tenant-123", + "UIPATH_TRACE_BASE_URL": trace_url, + "UIPATH_TRACING_ENABLED": "true", + "LOG_LEVEL": "DEBUG", + }, + working_directory=str(project), + ) + return response, output_file + + +def read_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +@pytest.mark.parametrize( + ("command", "extra_args"), + [ + ("run", []), + ("debug", ["--attach", "none"]), + ], +) +def test_server_runs_two_traced_jobs_after_trace_shutdown( + tmp_path: Path, + trace_stub: tuple[TraceServer, str], + command: str, + extra_args: list[str], +) -> None: + port = get_free_port() + start_cli_server_thread(port) + + trace_server, trace_url = trace_stub + project = tmp_path / "project" + project.mkdir() + write_project(project) + + job1_response, job1_output = asyncio.run( + run_traced_job(port, project, trace_url, "job-1", command, extra_args) + ) + job2_response, job2_output = asyncio.run( + run_traced_job(port, project, trace_url, "job-2", command, extra_args) + ) + + assert job1_response["success"] is True, job1_response + assert job2_response["success"] is True, job2_response + assert read_json(job1_output) == {"message": "ok: job-1"} + assert read_json(job2_output) == {"message": "ok: job-2"} + + result = read_json(project / "__uipath" / "output.json") + assert result["status"] == "successful" + assert "error" not in result + assert trace_server.posts diff --git a/packages/uipath/tests/cli/test_auth_server.py b/packages/uipath/tests/cli/test_auth_server.py new file mode 100644 index 000000000..f81d2c08b --- /dev/null +++ b/packages/uipath/tests/cli/test_auth_server.py @@ -0,0 +1,138 @@ +"""Security tests for the OAuth local callback server. + +Covers GHSA-32xc-7x5c-8vmf: the `/set_token` and `/log` endpoints must reject +unauthenticated POSTs (no / wrong OAuth `state`), and the server must bind to +loopback only rather than all interfaces. +""" + +import json +import os +import threading +import urllib.error +import urllib.request + +from uipath._cli._auth._auth_server import HTTPServer + +STATE = "LEGITIMATE_OAUTH_STATE_ABCDE12345" +CODE_VERIFIER = "LEGITIMATE_PKCE_CODE_VERIFIER" +DOMAIN = "cloud.uipath.com" + +ATTACKER_PAYLOAD = { + "access_token": "attacker-token", + "refresh_token": "attacker-refresh", + "expires_in": 3600, + "token_type": "Bearer", + "scope": "offline_access", +} + + +def _request(port, path, data, headers=None, method="POST"): + req = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=data, + headers=headers or {}, + method=method, + ) + try: + with urllib.request.urlopen(req, timeout=5) as resp: + return resp.status, resp.read().decode("utf-8") + except urllib.error.HTTPError as exc: + return exc.code, exc.read().decode("utf-8") + + +def _post(port, path, body, headers=None): + return _request( + port, + path, + json.dumps(body).encode("utf-8"), + {"Content-Type": "application/json", **(headers or {})}, + ) + + +async def test_endpoints_reject_unauthenticated_posts(tmp_path, monkeypatch): + """Only requests carrying the matching OAuth state are accepted. + + Exercises both /set_token and /log with missing, wrong, and valid state. + """ + monkeypatch.chdir(tmp_path) + + # Binding happens in create_server; the listen socket is up before the + # handler thread starts, so connections queue and no readiness sleep is + # needed. redirect_uri/client_id are required by the GET (index.html) path. + server = HTTPServer( + port=0, redirect_uri="http://localhost/callback", client_id="test-client" + ) + httpd = server.create_server(STATE, CODE_VERIFIER, DOMAIN) + port = httpd.server_address[1] + + results = {} + + def client(): + # DNS rebinding + results["rebind_get"] = _request( + port, "/", None, {"Host": "not-localhost.com"}, method="GET" + ) + results["rebind_post"] = _post( + port, + "/set_token", + ATTACKER_PAYLOAD, + {"X-Auth-State": STATE, "Host": "evil.com"}, + ) + # GET serves index.html with the OAuth params substituted in. + results["get"] = _request(port, "/anything", None, method="GET") + # /set_token: missing and wrong state are rejected. + results["set_missing"] = _post(port, "/set_token", ATTACKER_PAYLOAD) + results["set_wrong"] = _post( + port, "/set_token", ATTACKER_PAYLOAD, {"X-Auth-State": "not-the-state"} + ) + # Valid state but a non-JSON body -> graceful 400, not 500. + results["set_malformed"] = _request( + port, "/set_token", b"not json", {"X-Auth-State": STATE} + ) + # Unknown path -> 404. + results["unknown"] = _post(port, "/nope", {"x": 1}, {"X-Auth-State": STATE}) + # /log: missing and valid state. + results["log_missing"] = _post(port, "/log", {"msg": "x"}) + results["log_valid"] = _post( + port, "/log", {"msg": "x"}, {"X-Auth-State": STATE} + ) + # Valid /set_token last, to capture the token and unblock start(). + results["set_valid"] = _post( + port, "/set_token", {"access_token": "real"}, {"X-Auth-State": STATE} + ) + + t = threading.Thread(target=client, daemon=True) + t.start() + token_data = await server.start(STATE, CODE_VERIFIER, DOMAIN) + t.join(timeout=5) + + # DNS rebinding: forged Host is rejected on both GET and POST. + assert results["rebind_get"][0] == 403 + assert results["rebind_post"][0] == 403 + + # GET returns the page with the real state injected, placeholder gone. + assert results["get"][0] == 200 + assert STATE in results["get"][1] + assert "__PY_REPLACE_EXPECTED_STATE__" not in results["get"][1] + + assert results["set_missing"][0] == 403 + assert results["set_wrong"][0] == 403 + assert results["set_malformed"][0] == 400 + assert results["unknown"][0] == 404 + assert results["log_missing"][0] == 403 + assert results["log_valid"][0] == 200 + assert results["set_valid"][0] == 200 + + # Only the valid, state-bearing request was accepted. + assert token_data == {"access_token": "real"} + # The state-protected /log write happened for the valid request only. + assert os.path.exists(tmp_path / ".uipath" / ".error_log") + + +def test_server_binds_to_loopback_only(): + server = HTTPServer(port=0) + httpd = server.create_server(STATE, CODE_VERIFIER, DOMAIN) + try: + assert httpd.server_address[0] == "127.0.0.1" + finally: + httpd.server_close() diff --git a/packages/uipath/tests/cli/test_create_resources.py b/packages/uipath/tests/cli/test_create_resources.py new file mode 100644 index 000000000..aff1c7cab --- /dev/null +++ b/packages/uipath/tests/cli/test_create_resources.py @@ -0,0 +1,456 @@ +"""Unit tests for cli_push.create_resources virtual-resource fallback.""" + +import json +import os +from types import SimpleNamespace +from typing import Any, List, Optional +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + +from uipath._cli._utils._studio_project import ( + Status, + VirtualResourceResult, +) +from uipath.platform.errors import EnrichedException, FolderNotFoundException +from uipath.platform.resource_catalog import ResourceType + + +def _enriched_exc( + status_code: int = 404, body: bytes = b"not found" +) -> EnrichedException: + """Build an EnrichedException backed by a real HTTPStatusError.""" + request = httpx.Request("GET", "https://example.test/x") + response = httpx.Response(status_code, content=body, request=request) + http_err = httpx.HTTPStatusError("x", request=request, response=response) + return EnrichedException(http_err) + + +class _AsyncIterator: + """Minimal async iterator with aclose() to mimic resource_catalog pagination.""" + + def __init__(self, items: List[Any], raise_exc: Optional[Exception] = None): + self._items = iter(items) + self._raise_exc = raise_exc + self.aclose = AsyncMock() + + def __aiter__(self): + return self + + async def __anext__(self): + if self._raise_exc is not None: + raise self._raise_exc + try: + return next(self._items) + except StopIteration: + raise StopAsyncIteration from None + + +def _make_bindings(resources: List[dict[str, Any]]) -> str: + return json.dumps({"version": "2.2", "resources": resources}) + + +def _asset_binding( + name: str = "my_asset", + folder_path: str = "Shared", + metadata: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + return { + "resource": "asset", + "key": f"binding-{name}", + "value": { + "name": { + "defaultValue": name, + "isExpression": False, + "displayName": "name", + }, + "folderPath": { + "defaultValue": folder_path, + "isExpression": False, + "displayName": "folderPath", + }, + }, + "metadata": metadata, + } + + +def _found_resource( + key: str = "resource-key", + resource_type: str = "asset", + resource_sub_type: str = "stringAsset", + folder_path: str = "Shared", +) -> SimpleNamespace: + folder = SimpleNamespace( + key="folder-key", + fully_qualified_name=folder_path, + path=folder_path, + ) + return SimpleNamespace( + resource_key=key, + resource_type=resource_type, + resource_sub_type=resource_sub_type, + folders=[folder], + ) + + +@pytest.fixture +def bindings_file(tmp_path, monkeypatch): + """Write a bindings file to a tmp path and patch UiPathConfig.bindings_file_path.""" + path = tmp_path / "bindings.json" + + def _writer(content: str) -> str: + path.write_text(content, encoding="utf-8") + return str(path) + + from uipath.platform.common._config import ConfigurationManager + + monkeypatch.setattr( + ConfigurationManager, + "bindings_file_path", + property(lambda self: path), + ) + return _writer + + +@pytest.fixture +def mock_uipath(): + """Patch UiPath() in cli_push to return a mock with resource_catalog + connections. + + cli_push imports `UiPath` lazily from `uipath.platform` inside create_resources, + so we patch the source module. + """ + with patch("uipath.platform.UiPath") as mock_cls: + instance = MagicMock() + instance.resource_catalog = MagicMock() + instance.connections = MagicMock() + mock_cls.return_value = instance + yield instance + + +@pytest.fixture +def studio_client(): + from uipath._cli._utils._studio_project import ( + ResourceBuilderMetadataEntry, + ResourceBuilderMetadataVersion, + ) + + client = MagicMock() + client.create_referenced_resource = AsyncMock() + client.create_virtual_resource = AsyncMock() + supported = ResourceBuilderMetadataVersion(supportsInLineCreation=True) + # /metadata response — every kind our tests use supports inline creation. + client.get_resource_builder_metadata = AsyncMock( + return_value=[ + ResourceBuilderMetadataEntry(kind="asset", versions=[supported]), + ResourceBuilderMetadataEntry(kind="bucket", versions=[supported]), + ResourceBuilderMetadataEntry(kind="queue", versions=[supported]), + ResourceBuilderMetadataEntry(kind="taskCatalog", versions=[supported]), + ] + ) + return client + + +async def _run_create_resources(studio_client): + from uipath._cli.cli_push import create_resources + + await create_resources(studio_client) + + +async def test_catalog_hit_calls_referenced_resource_only( + bindings_file, mock_uipath, studio_client +): + bindings_file(_make_bindings([_asset_binding(metadata={"SubType": "stringAsset"})])) + mock_uipath.resource_catalog.list_by_type_async.return_value = _AsyncIterator( + [_found_resource()] + ) + studio_client.create_referenced_resource.return_value = SimpleNamespace( + status=Status.ADDED + ) + + await _run_create_resources(studio_client) + + studio_client.create_referenced_resource.assert_awaited_once() + studio_client.create_virtual_resource.assert_not_awaited() + + +async def test_catalog_miss_with_subtype_creates_virtual_with_type( + bindings_file, mock_uipath, studio_client +): + bindings_file(_make_bindings([_asset_binding(metadata={"SubType": "stringAsset"})])) + mock_uipath.resource_catalog.list_by_type_async.return_value = _AsyncIterator([]) + studio_client.create_virtual_resource.return_value = VirtualResourceResult( + status=Status.ADDED, + ) + + await _run_create_resources(studio_client) + + studio_client.create_virtual_resource.assert_awaited_once() + req = studio_client.create_virtual_resource.call_args.args[0] + assert req.kind == "asset" + assert req.name == "my_asset" + assert req.type == "stringAsset" + studio_client.create_referenced_resource.assert_not_awaited() + + +async def test_catalog_miss_without_subtype_creates_virtual_kind_only( + bindings_file, mock_uipath, studio_client +): + bindings_file(_make_bindings([_asset_binding(metadata=None)])) + mock_uipath.resource_catalog.list_by_type_async.return_value = _AsyncIterator([]) + studio_client.create_virtual_resource.return_value = VirtualResourceResult( + status=Status.ADDED, + ) + + await _run_create_resources(studio_client) + + studio_client.create_virtual_resource.assert_awaited_once() + req = studio_client.create_virtual_resource.call_args.args[0] + assert req.kind == "asset" + assert req.name == "my_asset" + assert req.type is None + # Body that will actually be sent excludes None → no "type" key. + body = req.model_dump(exclude_none=True) + assert "type" not in body + + +async def test_catalog_miss_metadata_without_subtype_key_creates_virtual_kind_only( + bindings_file, mock_uipath, studio_client +): + bindings_file(_make_bindings([_asset_binding(metadata={"Other": "x"})])) + mock_uipath.resource_catalog.list_by_type_async.return_value = _AsyncIterator([]) + studio_client.create_virtual_resource.return_value = VirtualResourceResult( + status=Status.ADDED, + ) + + await _run_create_resources(studio_client) + + req = studio_client.create_virtual_resource.call_args.args[0] + assert req.type is None + + +async def test_unknown_resource_type_skips_catalog_and_creates_virtual( + bindings_file, mock_uipath, studio_client +): + """Bindings with a resource kind unknown to ResourceType enum but supported + by the virtual endpoint (e.g. 'taskCatalog') should skip the resource + catalog lookup and fall through to the virtual fallback instead of raising + ValueError.""" + task_catalog_binding = { + "resource": "taskCatalog", + "key": "live.good.taskcatalog.Shared", + "value": { + "name": { + "defaultValue": "live.good.taskcatalog", + "isExpression": False, + "displayName": "Name", + }, + "folderPath": { + "defaultValue": "Shared", + "isExpression": False, + "displayName": "Folder Path", + }, + }, + "metadata": None, + } + bindings_file(_make_bindings([task_catalog_binding])) + studio_client.create_virtual_resource.return_value = VirtualResourceResult( + status=Status.ADDED, + ) + + await _run_create_resources(studio_client) + + mock_uipath.resource_catalog.list_by_type_async.assert_not_called() + studio_client.create_virtual_resource.assert_awaited_once() + req = studio_client.create_virtual_resource.call_args.args[0] + assert req.kind == "taskCatalog" + assert req.type is None + + +async def test_unsupported_virtual_kind_is_skipped_with_warning( + bindings_file, mock_uipath, studio_client +): + """Bindings whose kind the virtual endpoint cannot materialize (e.g. + 'choiceSet', 'webhook') should be skipped with a warning and + never reach create_virtual_resource.""" + choiceset_binding = { + "resource": "choiceSet", + "key": "live.good.choiceset.Shared", + "value": { + "name": { + "defaultValue": "live.good.choiceset", + "isExpression": False, + "displayName": "Name", + }, + "folderPath": { + "defaultValue": "Shared", + "isExpression": False, + "displayName": "Folder Path", + }, + }, + "metadata": None, + } + bindings_file(_make_bindings([choiceset_binding])) + + await _run_create_resources(studio_client) + + mock_uipath.resource_catalog.list_by_type_async.assert_not_called() + studio_client.create_virtual_resource.assert_not_awaited() + studio_client.create_referenced_resource.assert_not_awaited() + + +async def test_entity_binding_catalog_hit_creates_reference( + bindings_file, mock_uipath, studio_client +): + """Entity bindings should go through the resource catalog lookup. + When found, a referenced resource should be created.""" + entity_binding = { + "resource": "entity", + "key": "live.good.entity.Shared", + "value": { + "name": { + "defaultValue": "live.good.entity", + "isExpression": False, + "displayName": "Name", + }, + "folderPath": { + "defaultValue": "Shared", + "isExpression": False, + "displayName": "Folder Path", + }, + }, + "metadata": None, + } + bindings_file(_make_bindings([entity_binding])) + mock_uipath.resource_catalog.list_by_type_async.return_value = _AsyncIterator( + [_found_resource(resource_type="entity", resource_sub_type="Native")] + ) + studio_client.create_referenced_resource.return_value = SimpleNamespace( + status=Status.ADDED + ) + + await _run_create_resources(studio_client) + + mock_uipath.resource_catalog.list_by_type_async.assert_called_once_with( + resource_type=ResourceType.ENTITY, + name="live.good.entity", + folder_path="Shared", + ) + studio_client.create_referenced_resource.assert_awaited_once() + studio_client.create_virtual_resource.assert_not_awaited() + + +async def test_folder_not_found_falls_back_to_virtual( + bindings_file, mock_uipath, studio_client +): + bindings_file(_make_bindings([_asset_binding(metadata={"SubType": "stringAsset"})])) + mock_uipath.resource_catalog.list_by_type_async.return_value = _AsyncIterator( + [], raise_exc=FolderNotFoundException("missing folder") + ) + studio_client.create_virtual_resource.return_value = VirtualResourceResult( + status=Status.ADDED, + ) + + await _run_create_resources(studio_client) + + studio_client.create_virtual_resource.assert_awaited_once() + req = studio_client.create_virtual_resource.call_args.args[0] + assert req.kind == "asset" + assert req.type == "stringAsset" + + +async def test_virtual_enriched_exception_caught_and_logged_as_warning( + bindings_file, mock_uipath, studio_client +): + bindings_file( + _make_bindings( + [ + _asset_binding(name="first"), + _asset_binding(name="second"), + ] + ) + ) + mock_uipath.resource_catalog.list_by_type_async.return_value = _AsyncIterator([]) + # First binding raises, second succeeds → loop must continue past the raise. + studio_client.create_virtual_resource.side_effect = [ + _enriched_exc(status_code=500, body=b"boom"), + VirtualResourceResult(status=Status.ADDED), + ] + + await _run_create_resources(studio_client) + + assert studio_client.create_virtual_resource.await_count == 2 + + +async def test_connection_branch_unchanged_no_virtual_fallback( + bindings_file, mock_uipath, studio_client +): + """Connection bindings retain old behavior: retrieve_async + warn on miss, no virtual.""" + connection_binding = { + "resource": "connection", + "key": "binding-conn", + "value": { + "ConnectionId": { + "defaultValue": "missing-conn-id", + "isExpression": False, + "displayName": "ConnectionId", + } + }, + "metadata": {"Connector": "salesforce"}, + } + bindings_file(_make_bindings([connection_binding])) + mock_uipath.connections.retrieve_async = AsyncMock(side_effect=_enriched_exc()) + + await _run_create_resources(studio_client) + + mock_uipath.connections.retrieve_async.assert_awaited_once() + studio_client.create_virtual_resource.assert_not_awaited() + studio_client.create_referenced_resource.assert_not_awaited() + + +async def test_guardrail_binding_without_folder_path_is_skipped( + bindings_file, mock_uipath, studio_client +): + # No folderPath in value → guardrail; should be skipped entirely. + guardrail_binding = { + "resource": "asset", + "key": "binding-guard", + "value": { + "name": { + "defaultValue": "g", + "isExpression": False, + "displayName": "name", + } + }, + "metadata": None, + } + bindings_file(_make_bindings([guardrail_binding])) + + await _run_create_resources(studio_client) + + mock_uipath.resource_catalog.list_by_type_async.assert_not_called() + studio_client.create_virtual_resource.assert_not_awaited() + studio_client.create_referenced_resource.assert_not_awaited() + + +# Ensure env doesn't leak solution ids between tests. +@pytest.fixture(autouse=True) +def _reset_solution_id(): + from uipath.platform.common._config import ConfigurationManager + + ConfigurationManager.studio_solution_id = None + yield + ConfigurationManager.studio_solution_id = None + + +# Set minimal env so UiPath() construction inside create_resources (if not mocked +# away cleanly) doesn't trip on missing creds. The mock_uipath fixture patches +# the class, so this is defense-in-depth. +@pytest.fixture(autouse=True) +def _env(monkeypatch): + monkeypatch.setenv("UIPATH_URL", "https://cloud.uipath.com/org/tenant") + monkeypatch.setenv("UIPATH_ACCESS_TOKEN", "mock_token") + yield + for k in ("UIPATH_URL", "UIPATH_ACCESS_TOKEN"): + if k in os.environ: + monkeypatch.delenv(k, raising=False) diff --git a/packages/uipath/tests/cli/test_debug_bridge_selection.py b/packages/uipath/tests/cli/test_debug_bridge_selection.py new file mode 100644 index 000000000..732461c77 --- /dev/null +++ b/packages/uipath/tests/cli/test_debug_bridge_selection.py @@ -0,0 +1,64 @@ +"""Tests for `get_debug_bridge()` selection matrix. + +Locks in the non-breaking-change contract: absence of `attach` preserves the +legacy `job_id`-based selection. Explicit `attach` overrides that selection. +""" + +from __future__ import annotations + +import pytest + +from uipath._cli._debug._bridge import ( + ConsoleDebugBridge, + SignalRDebugBridge, + get_debug_bridge, +) +from uipath.runtime import UiPathRuntimeContext +from uipath.runtime.debug import DetachedDebugBridge + + +def _ctx(**overrides) -> UiPathRuntimeContext: + return UiPathRuntimeContext(**overrides) + + +def test_attach_none_returns_detached_bridge_without_job_id(): + bridge = get_debug_bridge(_ctx(), attach="none") + assert isinstance(bridge, DetachedDebugBridge) + + +def test_attach_none_returns_detached_bridge_even_when_job_id_set(monkeypatch): + """'none' wins over job_id — this is the whole point of the flag.""" + monkeypatch.setenv("UIPATH_URL", "https://cloud.uipath.com") + bridge = get_debug_bridge(_ctx(job_id="job-123"), attach="none") + assert isinstance(bridge, DetachedDebugBridge) + + +def test_attach_signalr_forces_signalr_bridge(monkeypatch): + monkeypatch.setenv("UIPATH_URL", "https://cloud.uipath.com") + bridge = get_debug_bridge(_ctx(job_id="job-123"), attach="signalr") + assert isinstance(bridge, SignalRDebugBridge) + + +def test_attach_console_forces_console_bridge_even_when_job_id_set(monkeypatch): + monkeypatch.setenv("UIPATH_URL", "https://cloud.uipath.com") + bridge = get_debug_bridge(_ctx(job_id="job-123"), attach="console") + assert isinstance(bridge, ConsoleDebugBridge) + + +def test_legacy_selection_signalr_when_job_id_set_and_no_attach(monkeypatch): + """Non-breaking change assertion: absence of `attach` preserves today's behavior.""" + monkeypatch.setenv("UIPATH_URL", "https://cloud.uipath.com") + bridge = get_debug_bridge(_ctx(job_id="job-123")) + assert isinstance(bridge, SignalRDebugBridge) + + +def test_legacy_selection_console_when_no_job_id_and_no_attach(): + """Non-breaking change assertion: absence of `attach` preserves today's behavior.""" + bridge = get_debug_bridge(_ctx()) + assert isinstance(bridge, ConsoleDebugBridge) + + +def test_attach_signalr_without_job_id_raises(): + """Explicit signalr without job_id is a user error — surface it loudly.""" + with pytest.raises(ValueError, match="UIPATH_URL and UIPATH_JOB_KEY"): + get_debug_bridge(_ctx(), attach="signalr") diff --git a/packages/uipath/tests/cli/test_debug_simulation.py b/packages/uipath/tests/cli/test_debug_simulation.py index b2d795c79..9185d98c6 100644 --- a/packages/uipath/tests/cli/test_debug_simulation.py +++ b/packages/uipath/tests/cli/test_debug_simulation.py @@ -82,11 +82,12 @@ def test_loads_valid_simulation_config( assert result is not None assert isinstance(result, MockingContext) assert result.name == "debug-simulation" - assert result.strategy is not None + # Legacy format routes to local LLM mocker via strategy + assert result.components is None or len(result.components) == 0 assert isinstance(result.strategy, LLMMockingStrategy) - assert result.strategy.prompt == valid_simulation_config["instructions"] assert len(result.strategy.tools_to_simulate) == 3 assert result.strategy.tools_to_simulate[0].name == "Web Reader" + assert result.strategy.prompt == valid_simulation_config["instructions"] def test_returns_none_when_disabled( self, temp_dir: str, disabled_simulation_config: dict[str, Any] @@ -241,6 +242,9 @@ def test_debug_always_wraps_with_mock_runtime( ) as mock_factory_get: mock_runtime = Mock() mock_runtime.dispose = AsyncMock() + mock_runtime.get_schema = AsyncMock( + return_value=Mock(metadata=None) + ) mock_factory = Mock() mock_factory.new_runtime = AsyncMock(return_value=mock_runtime) @@ -305,6 +309,9 @@ def test_debug_wraps_with_mock_runtime_on_error( ) as mock_factory_get: mock_runtime = Mock() mock_runtime.dispose = AsyncMock() + mock_runtime.get_schema = AsyncMock( + return_value=Mock(metadata=None) + ) mock_factory = Mock() mock_factory.new_runtime = AsyncMock(return_value=mock_runtime) @@ -361,6 +368,225 @@ def test_simulation_config_enables_tool_mocking( # Clean up clear_execution_context() + +_SIMULATION_JSON = { + "enabled": True, + "toolsToSimulate": [{"name": "check_syntax"}, {"name": "check_style"}], + "instructions": "Simulate.", +} + +_COMPONENT_SIMULATION_JSON = { + "enabled": True, + "components": [ + { + "componentId": "get_current_weather", + "componentType": "tool", + "simulationStrategy": 0, + "simulationInstruction": "Return realistic weather data", + }, + { + "componentId": "get_forecast", + "componentType": "tool", + "simulationStrategy": 0, + "simulationInstruction": "Return a multi-day forecast", + }, + ], +} + + +class TestDebugSimulationFlag: + """Tests for the --simulation flag on the debug command.""" + + def _make_debug_patches(self): + """Create common mock objects for debug command tests.""" + mock_runtime = Mock() + mock_runtime.dispose = AsyncMock() + mock_runtime.get_schema = AsyncMock(return_value=Mock(metadata=None)) + + mock_factory = Mock() + mock_factory.new_runtime = AsyncMock(return_value=mock_runtime) + mock_factory.get_settings = AsyncMock(return_value=Mock(trace_settings=None)) + mock_factory.dispose = AsyncMock() + + mock_debug_runtime = Mock() + mock_debug_runtime.dispose = AsyncMock() + + return mock_factory, mock_runtime, mock_debug_runtime + + def test_invalid_simulation_json_exits_with_error( + self, runner: CliRunner, temp_dir: str + ): + with runner.isolated_filesystem(temp_dir=temp_dir): + with open("uipath.json", "w") as f: + json.dump({"functions": {"main": "main.py:main"}}, f) + with open("main.py", "w") as f: + f.write("async def main(input): return {}") + + result = runner.invoke( + cli, ["debug", "main", "--simulation", "{ not valid json }"] + ) + assert result.exit_code == 1 + assert "Invalid" in result.output + + def test_simulation_flag_wraps_runtime_with_mock_runtime( + self, runner: CliRunner, temp_dir: str + ): + mock_factory, mock_runtime, mock_debug_runtime = self._make_debug_patches() + + with runner.isolated_filesystem(temp_dir=temp_dir): + with open("uipath.json", "w") as f: + json.dump({"functions": {"main": "main.py:main"}}, f) + with open("main.py", "w") as f: + f.write("async def main(input): return {}") + + with ( + patch( + "uipath._cli.cli_debug.Middlewares.next", + return_value=MiddlewareResult( + should_continue=True, + info_message=None, + error_message=None, + should_include_stacktrace=False, + ), + ), + patch( + "uipath._cli.cli_debug.UiPathRuntimeFactoryRegistry.get", + return_value=mock_factory, + ), + patch("uipath._cli.cli_debug.get_debug_bridge"), + patch( + "uipath._cli.cli_debug.UiPathDebugRuntime", + return_value=mock_debug_runtime, + ), + patch("uipath._cli.cli_debug.UiPathMockRuntime") as mock_cls, + ): + mock_cls.return_value = Mock( + execute=AsyncMock( + return_value=Mock(status="SUCCESSFUL", output={}) + ), + dispose=AsyncMock(), + ) + runner.invoke( + cli, + [ + "debug", + "main", + "{}", + "--simulation", + json.dumps(_SIMULATION_JSON), + ], + ) + + assert mock_cls.called + assert mock_cls.call_args.kwargs["mocking_context"] is not None + assert mock_cls.call_args.kwargs["delegate"] is mock_debug_runtime + + def test_simulation_flag_disabled_does_not_wrap_runtime( + self, runner: CliRunner, temp_dir: str + ): + mock_factory, mock_runtime, mock_debug_runtime = self._make_debug_patches() + disabled = {**_SIMULATION_JSON, "enabled": False} + + with runner.isolated_filesystem(temp_dir=temp_dir): + with open("uipath.json", "w") as f: + json.dump({"functions": {"main": "main.py:main"}}, f) + with open("main.py", "w") as f: + f.write("async def main(input): return {}") + + with ( + patch( + "uipath._cli.cli_debug.Middlewares.next", + return_value=MiddlewareResult( + should_continue=True, + info_message=None, + error_message=None, + should_include_stacktrace=False, + ), + ), + patch( + "uipath._cli.cli_debug.UiPathRuntimeFactoryRegistry.get", + return_value=mock_factory, + ), + patch("uipath._cli.cli_debug.get_debug_bridge"), + patch( + "uipath._cli.cli_debug.UiPathDebugRuntime", + return_value=mock_debug_runtime, + ), + patch("uipath._cli.cli_debug.UiPathMockRuntime") as mock_cls, + patch( + "uipath._cli.cli_debug.load_simulation_config", + return_value=None, + ), + ): + mock_debug_runtime.execute = AsyncMock( + return_value=Mock(status="SUCCESSFUL", output={}) + ) + runner.invoke( + cli, + ["debug", "main", "{}", "--simulation", json.dumps(disabled)], + ) + + assert not mock_cls.called + + def test_simulation_flag_with_component_format( + self, runner: CliRunner, temp_dir: str + ): + """Test that --simulation with new component format sets components on MockingContext.""" + mock_factory, mock_runtime, mock_debug_runtime = self._make_debug_patches() + + with runner.isolated_filesystem(temp_dir=temp_dir): + with open("uipath.json", "w") as f: + json.dump({"functions": {"main": "main.py:main"}}, f) + with open("main.py", "w") as f: + f.write("async def main(input): return {}") + + with ( + patch( + "uipath._cli.cli_debug.Middlewares.next", + return_value=MiddlewareResult( + should_continue=True, + info_message=None, + error_message=None, + should_include_stacktrace=False, + ), + ), + patch( + "uipath._cli.cli_debug.UiPathRuntimeFactoryRegistry.get", + return_value=mock_factory, + ), + patch("uipath._cli.cli_debug.get_debug_bridge"), + patch( + "uipath._cli.cli_debug.UiPathDebugRuntime", + return_value=mock_debug_runtime, + ), + patch("uipath._cli.cli_debug.UiPathMockRuntime") as mock_cls, + ): + mock_cls.return_value = Mock( + execute=AsyncMock( + return_value=Mock(status="SUCCESSFUL", output={}) + ), + dispose=AsyncMock(), + ) + runner.invoke( + cli, + [ + "debug", + "main", + "{}", + "--simulation", + json.dumps(_COMPONENT_SIMULATION_JSON), + ], + ) + + assert mock_cls.called + mocking_context = mock_cls.call_args.kwargs["mocking_context"] + assert mocking_context is not None + assert mocking_context.components is not None + assert len(mocking_context.components) == 2 + assert mocking_context.components[0].component_id == "get_current_weather" + assert mocking_context.components[1].component_id == "get_forecast" + assert mocking_context.strategy is None + def test_middleware_short_circuits_before_mock_runtime( self, runner: CliRunner, @@ -416,6 +642,34 @@ def test_enabled_defaults_to_true_when_missing(self, temp_dir: str): # Should load successfully since enabled defaults to true assert result is not None + def test_new_format_loads_components(self, temp_dir: str): + """Test that new per-component format routes to API-based mocker (components set).""" + config = { + "enabled": True, + "components": [ + { + "componentId": "my_tool", + "componentType": "tool", + "simulationStrategy": 0, + "simulationInstruction": "Simulate this tool", + } + ], + } + simulation_path = Path(temp_dir) / "simulation.json" + with open(simulation_path, "w", encoding="utf-8") as f: + json.dump(config, f) + + with patch(f"{MOCK_RUNTIME_PATCH_PATH}.Path.cwd", return_value=Path(temp_dir)): + result = load_simulation_config() + + assert result is not None + assert isinstance(result, MockingContext) + # New format: components set, strategy is None + assert result.components is not None + assert len(result.components) == 1 + assert result.components[0].component_id == "my_tool" + assert result.strategy is None + def test_handles_tool_name_normalization(self, temp_dir: str): """Test that tool names with underscores work correctly.""" config = { diff --git a/packages/uipath/tests/cli/test_dev.py b/packages/uipath/tests/cli/test_dev.py new file mode 100644 index 000000000..8b2a93c04 --- /dev/null +++ b/packages/uipath/tests/cli/test_dev.py @@ -0,0 +1,70 @@ +import sys +import types +from unittest.mock import AsyncMock, MagicMock + +import pytest +from click.testing import CliRunner + +from uipath._cli import cli, cli_dev +from uipath._cli.middlewares import MiddlewareResult +from uipath.platform.common import UiPathExecutionContext + + +def test_create_dev_context_and_factory_uses_dev_command( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The helper builds a 'dev' context (source 'playground') and its factory.""" + sentinel_factory = MagicMock(name="factory") + captured: dict[str, object] = {} + + def fake_get(context: object) -> object: + captured["command"] = context.command # type: ignore[attr-defined] + return sentinel_factory + + monkeypatch.setattr( + "uipath._cli.cli_dev.UiPathRuntimeFactoryRegistry.get", fake_get + ) + + context, factory = cli_dev._create_dev_context_and_factory(None) # type: ignore[arg-type] + + assert factory is sentinel_factory + assert captured["command"] == "dev" + assert context.execution_source == "playground" + + +def test_dev_terminal_sets_execution_source_during_run( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Running `dev terminal` scopes the execution source to 'playground'.""" + seen: dict[str, object] = {} + + async def fake_run_async() -> None: + seen["source"] = UiPathExecutionContext().execution_source + + fake_console = MagicMock() + fake_console.run_async = fake_run_async + + fake_module = types.ModuleType("uipath.dev") + fake_module.UiPathDeveloperConsole = MagicMock(return_value=fake_console) # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "uipath.dev", fake_module) + + mock_factory = MagicMock() + mock_factory.dispose = AsyncMock() + + monkeypatch.setattr(cli_dev, "_check_dev_dependency", lambda interface: None) + monkeypatch.setattr(cli_dev, "setup_debugging", lambda debug, port: True) + monkeypatch.setattr( + "uipath._cli.cli_dev.Middlewares.next", + lambda *a, **k: MiddlewareResult(should_continue=True), + ) + monkeypatch.setattr( + "uipath._cli.cli_dev.UiPathRuntimeFactoryRegistry.get", + lambda context: mock_factory, + ) + + result = CliRunner().invoke(cli, ["dev", "terminal"]) + + assert result.exit_code == 0, result.output + assert seen["source"] == "playground" + # token released once the run completes + assert UiPathExecutionContext().execution_source is None diff --git a/packages/uipath/tests/cli/test_governance_bootstrap.py b/packages/uipath/tests/cli/test_governance_bootstrap.py new file mode 100644 index 000000000..fabe3650c --- /dev/null +++ b/packages/uipath/tests/cli/test_governance_bootstrap.py @@ -0,0 +1,858 @@ +"""Tests replace runtime-governance types via ``monkeypatch.setattr`` on +the bootstrap module's namespace (not via ``sys.modules``) — the +bootstrap imports them at top level. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from uipath._cli._governance_bootstrap import ( + GovernanceBootstrap, + resolve_governance, +) +from uipath.core.governance import EnforcementMode, PolicyContext +from uipath.runtime.governance.native.models import PolicyIndex +from uipath.runtime.governance.runtime import UiPathGovernedRuntime + + +@pytest.fixture +def cwd(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Change into an isolated cwd for each test. + + Kept even though ``resolve_governance`` no longer touches project + files — several tests still want an isolated working directory so + incidental file access (SDK config discovery, logging) doesn't leak + across cases. + """ + monkeypatch.chdir(tmp_path) + return tmp_path + + +def _install_fake_runtime_governance( + monkeypatch: pytest.MonkeyPatch, + *, + audit_manager_cls: type, + metadata_cls: type, + evaluator_cls: type, + compensator_cls: type, +) -> None: + """Replace the runtime-governance names bound in the bootstrap module.""" + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.AuditManager", + audit_manager_cls, + ) + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.GovernanceRuntimeMetadata", + metadata_cls, + ) + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.GovernanceEvaluator", + evaluator_cls, + ) + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.GuardrailCompensator", + compensator_cls, + ) + + +class _FakeAuditManager: + """Records the constructor kwargs so tests can inspect them.""" + + def __init__(self, *, track_event: Any, runtime_metadata: Any) -> None: + self.track_event = track_event + self.runtime_metadata = runtime_metadata + + +class _FakeMetadata: + def __init__(self, *, agent_type: str | None, agent_framework: str) -> None: + self.agent_type = agent_type + self.agent_framework = agent_framework + + +class _FakeCompensator: + def __init__(self, provider: Any) -> None: + self.provider = provider + + +class _FakeEvaluator: + def __init__( + self, + policy_index: Any, + *, + enforcement_mode: Any, + audit_manager: Any, + compensator: Any, + ) -> None: + self.policy_index = policy_index + self.enforcement_mode = enforcement_mode + self.audit_manager = audit_manager + self.compensator = compensator + + +def _fake_policy_response(*, mode: EnforcementMode | None, policies: str) -> MagicMock: + resp = MagicMock() + resp.mode = mode + resp.policies = policies + return resp + + +def _stub_provider( + monkeypatch: pytest.MonkeyPatch, *, response_or_exc: Any +) -> MagicMock: + """Replace the ``UiPath()`` + provider construction with a stub whose + ``get_policy_async`` returns ``response_or_exc`` (or raises if it's + an exception instance). + """ + provider = MagicMock() + if isinstance(response_or_exc, BaseException): + provider.get_policy_async = AsyncMock(side_effect=response_or_exc) + else: + provider.get_policy_async = AsyncMock(return_value=response_or_exc) + provider.track_event_async = AsyncMock() + + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.UiPath", + lambda: MagicMock(governance=MagicMock()), + ) + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.UiPathPlatformGovernanceProvider", + lambda service: provider, + ) + return provider + + +class TestResolveGovernance: + async def test_returns_none_when_feature_flag_disabled( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.is_governance_enabled", + lambda: False, + ) + assert ( + await resolve_governance( + agent_framework="langgraph", + agent_type="uipath_coded", + is_conversational=False, + ) + is None + ) + + @pytest.mark.parametrize("is_conversational", [True, False]) + async def test_is_conversational_forwarded_verbatim_to_policy_context( + self, + monkeypatch: pytest.MonkeyPatch, + cwd: Path, + is_conversational: bool, + ) -> None: + """The caller derives ``is_conversational`` from + ``bool(ctx.conversation_id)`` and passes it through. The + bootstrap must forward the exact value to :class:`PolicyContext` + so the backend can select the conversational or autonomous + policy view.""" + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.is_governance_enabled", + lambda: True, + ) + _install_fake_runtime_governance( + monkeypatch, + audit_manager_cls=_FakeAuditManager, + metadata_cls=_FakeMetadata, + evaluator_cls=_FakeEvaluator, + compensator_cls=_FakeCompensator, + ) + provider = _stub_provider( + monkeypatch, + response_or_exc=_fake_policy_response( + mode=EnforcementMode.ENFORCE, policies="rules: []" + ), + ) + + result = await resolve_governance( + agent_framework="langgraph", + agent_type="uipath_coded", + is_conversational=is_conversational, + ) + assert result is not None + try: + provider.get_policy_async.assert_awaited_once() + context_arg = provider.get_policy_async.await_args.args[0] + assert isinstance(context_arg, PolicyContext) + assert context_arg.is_conversational is is_conversational + finally: + result.dispose() + + async def test_returns_none_when_policy_fetch_fails( + self, + monkeypatch: pytest.MonkeyPatch, + cwd: Path, + ) -> None: + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.is_governance_enabled", + lambda: True, + ) + _install_fake_runtime_governance( + monkeypatch, + audit_manager_cls=_FakeAuditManager, + metadata_cls=_FakeMetadata, + evaluator_cls=_FakeEvaluator, + compensator_cls=_FakeCompensator, + ) + _stub_provider(monkeypatch, response_or_exc=RuntimeError("backend unreachable")) + assert ( + await resolve_governance( + agent_framework="langgraph", + agent_type="uipath_coded", + is_conversational=False, + ) + is None + ) + + async def test_returns_none_when_mode_is_disabled( + self, + monkeypatch: pytest.MonkeyPatch, + cwd: Path, + ) -> None: + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.is_governance_enabled", + lambda: True, + ) + _install_fake_runtime_governance( + monkeypatch, + audit_manager_cls=_FakeAuditManager, + metadata_cls=_FakeMetadata, + evaluator_cls=_FakeEvaluator, + compensator_cls=_FakeCompensator, + ) + _stub_provider( + monkeypatch, + response_or_exc=_fake_policy_response( + mode=EnforcementMode.DISABLED, policies="rules:" + ), + ) + assert ( + await resolve_governance( + agent_framework="langgraph", + agent_type="uipath_coded", + is_conversational=False, + ) + is None + ) + + async def test_returns_none_when_mode_is_none( + self, + monkeypatch: pytest.MonkeyPatch, + cwd: Path, + ) -> None: + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.is_governance_enabled", + lambda: True, + ) + _install_fake_runtime_governance( + monkeypatch, + audit_manager_cls=_FakeAuditManager, + metadata_cls=_FakeMetadata, + evaluator_cls=_FakeEvaluator, + compensator_cls=_FakeCompensator, + ) + _stub_provider( + monkeypatch, + response_or_exc=_fake_policy_response(mode=None, policies="rules:"), + ) + assert ( + await resolve_governance( + agent_framework="langgraph", + agent_type="uipath_coded", + is_conversational=False, + ) + is None + ) + + async def test_returns_none_when_policies_empty( + self, + monkeypatch: pytest.MonkeyPatch, + cwd: Path, + ) -> None: + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.is_governance_enabled", + lambda: True, + ) + _install_fake_runtime_governance( + monkeypatch, + audit_manager_cls=_FakeAuditManager, + metadata_cls=_FakeMetadata, + evaluator_cls=_FakeEvaluator, + compensator_cls=_FakeCompensator, + ) + _stub_provider( + monkeypatch, + response_or_exc=_fake_policy_response( + mode=EnforcementMode.ENFORCE, policies="" + ), + ) + assert ( + await resolve_governance( + agent_framework="langgraph", + agent_type="uipath_coded", + is_conversational=False, + ) + is None + ) + + async def test_returns_none_when_policy_compilation_fails( + self, + monkeypatch: pytest.MonkeyPatch, + cwd: Path, + ) -> None: + """Malformed YAML must be caught by the compile-time try/except so + governance skips cleanly rather than propagating a ``YAMLError`` + out of ``resolve_governance``. + """ + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.is_governance_enabled", + lambda: True, + ) + _install_fake_runtime_governance( + monkeypatch, + audit_manager_cls=_FakeAuditManager, + metadata_cls=_FakeMetadata, + evaluator_cls=_FakeEvaluator, + compensator_cls=_FakeCompensator, + ) + _stub_provider( + monkeypatch, + response_or_exc=_fake_policy_response( + # Unclosed flow mapping — ``yaml.safe_load_all`` raises + # ``ScannerError`` (a subclass of ``YAMLError``), which + # ``resolve_governance``'s try/except converts to + # ``None``. + mode=EnforcementMode.ENFORCE, + policies="foo: {unclosed", + ), + ) + assert ( + await resolve_governance( + agent_framework="langgraph", + agent_type="uipath_coded", + is_conversational=False, + ) + is None + ) + + async def test_success_returns_bootstrap_with_dispose_contract( + self, + monkeypatch: pytest.MonkeyPatch, + cwd: Path, + ) -> None: + """Happy path -- named fields populated and dispose unregisters + atexit + shuts the dispatcher down. + """ + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.is_governance_enabled", + lambda: True, + ) + _install_fake_runtime_governance( + monkeypatch, + audit_manager_cls=_FakeAuditManager, + metadata_cls=_FakeMetadata, + evaluator_cls=_FakeEvaluator, + compensator_cls=_FakeCompensator, + ) + _stub_provider( + monkeypatch, + response_or_exc=_fake_policy_response( + mode=EnforcementMode.ENFORCE, policies="rules: [a, b]" + ), + ) + + # Capture what the code hands to atexit so we can verify + # register/unregister without relying on atexit internals. + registered: list[Any] = [] + + def _fake_register(func: Any, *_a: Any, **_kw: Any) -> Any: + registered.append(func) + return func + + def _fake_unregister(func: Any) -> None: + if func in registered: + registered.remove(func) + + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.atexit.register", _fake_register + ) + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.atexit.unregister", _fake_unregister + ) + + result = await resolve_governance( + agent_framework="langgraph", + agent_type="uipath_coded", + is_conversational=False, + ) + assert result is not None + + assert isinstance(result.evaluator, _FakeEvaluator) + assert isinstance(result.policy_index, PolicyIndex) + assert result.policy_index.total_rules == 0 + assert result.enforcement_mode == EnforcementMode.ENFORCE + assert isinstance(result.evaluator.audit_manager, _FakeAuditManager) + assert isinstance(result.evaluator.compensator, _FakeCompensator) + assert callable(result.evaluator.audit_manager.track_event) + + assert ( + result.evaluator.audit_manager.runtime_metadata.agent_type == "uipath_coded" + ) + assert ( + result.evaluator.audit_manager.runtime_metadata.agent_framework + == "langgraph" + ) + + assert len(registered) == 1 + result.dispose() + assert not registered, "atexit hook was not unregistered on dispose" + result.dispose() # idempotent + + async def test_agent_type_forwarded_verbatim_to_metadata( + self, + monkeypatch: pytest.MonkeyPatch, + cwd: Path, + ) -> None: + """The ``agent_type`` argument is forwarded verbatim to + :class:`GovernanceRuntimeMetadata` -- the CLI does not classify + the project; the factory does via + :attr:`UiPathRuntimeFactorySettings.agent_type`. + """ + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.is_governance_enabled", + lambda: True, + ) + _install_fake_runtime_governance( + monkeypatch, + audit_manager_cls=_FakeAuditManager, + metadata_cls=_FakeMetadata, + evaluator_cls=_FakeEvaluator, + compensator_cls=_FakeCompensator, + ) + _stub_provider( + monkeypatch, + response_or_exc=_fake_policy_response( + mode=EnforcementMode.ENFORCE, policies="rules: [a, b]" + ), + ) + + result = await resolve_governance( + agent_framework="lowcode", + agent_type="uipath_lowcode", + is_conversational=False, + ) + assert result is not None + try: + assert isinstance(result.evaluator, _FakeEvaluator) + assert ( + result.evaluator.audit_manager.runtime_metadata.agent_type + == "uipath_lowcode" + ) + assert ( + result.evaluator.audit_manager.runtime_metadata.agent_framework + == "lowcode" + ) + finally: + result.dispose() + + async def test_agent_framework_none_passes_through_as_unknown( + self, + monkeypatch: pytest.MonkeyPatch, + cwd: Path, + ) -> None: + """A factory with no ``agent_framework`` opinion emits + ``"unknown"`` -- symmetric with the ``agent_type`` fallback.""" + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.is_governance_enabled", + lambda: True, + ) + _install_fake_runtime_governance( + monkeypatch, + audit_manager_cls=_FakeAuditManager, + metadata_cls=_FakeMetadata, + evaluator_cls=_FakeEvaluator, + compensator_cls=_FakeCompensator, + ) + _stub_provider( + monkeypatch, + response_or_exc=_fake_policy_response( + mode=EnforcementMode.ENFORCE, policies="rules: []" + ), + ) + + result = await resolve_governance( + agent_framework=None, + agent_type="uipath_coded", + is_conversational=False, + ) + assert result is not None + try: + assert isinstance(result.evaluator, _FakeEvaluator) + assert ( + result.evaluator.audit_manager.runtime_metadata.agent_framework + == "unknown" + ) + finally: + result.dispose() + + async def test_agent_type_none_passes_through_to_metadata( + self, + monkeypatch: pytest.MonkeyPatch, + cwd: Path, + ) -> None: + """A factory with no ``agent_type`` opinion yields ``None`` on + the metadata -- the backend decides how to interpret the gap.""" + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.is_governance_enabled", + lambda: True, + ) + _install_fake_runtime_governance( + monkeypatch, + audit_manager_cls=_FakeAuditManager, + metadata_cls=_FakeMetadata, + evaluator_cls=_FakeEvaluator, + compensator_cls=_FakeCompensator, + ) + _stub_provider( + monkeypatch, + response_or_exc=_fake_policy_response( + mode=EnforcementMode.ENFORCE, policies="rules: []" + ), + ) + + result = await resolve_governance( + agent_framework="langgraph", + agent_type=None, + is_conversational=False, + ) + assert result is not None + try: + assert isinstance(result.evaluator, _FakeEvaluator) + # Metadata field is strict ``str`` with default ``"unknown"`` + # -- the bootstrap forwards that when the factory has no + # opinion, so the CLI never has to invent a value. + assert ( + result.evaluator.audit_manager.runtime_metadata.agent_type == "unknown" + ) + finally: + result.dispose() + + async def test_wrap_runtime_produces_governed_runtime_with_bootstrap_fields( + self, + monkeypatch: pytest.MonkeyPatch, + cwd: Path, + ) -> None: + """``GovernanceBootstrap.wrap_runtime`` must construct a + :class:`UiPathGovernedRuntime` populated from the bootstrap's + own ``evaluator`` / ``policy_index`` / ``enforcement_mode`` plus + the caller's ``agent_name`` / ``runtime_id``. This is the code + path CLI callers replaced their manual construction with. + """ + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.is_governance_enabled", + lambda: True, + ) + _install_fake_runtime_governance( + monkeypatch, + audit_manager_cls=_FakeAuditManager, + metadata_cls=_FakeMetadata, + evaluator_cls=_FakeEvaluator, + compensator_cls=_FakeCompensator, + ) + _stub_provider( + monkeypatch, + response_or_exc=_fake_policy_response( + mode=EnforcementMode.ENFORCE, policies="rules: []" + ), + ) + + result = await resolve_governance( + agent_framework="langgraph", + agent_type="uipath_coded", + is_conversational=False, + ) + assert isinstance(result, GovernanceBootstrap) + try: + delegate = MagicMock() # stand-in for the real runtime + wrapped = result.wrap_runtime( + delegate, + agent_name="my-agent", + runtime_id="run-123", + ) + assert isinstance(wrapped, UiPathGovernedRuntime) + assert wrapped._delegate is delegate + assert wrapped._policy_index is result.policy_index + assert wrapped._enforcement_mode is result.enforcement_mode + assert wrapped._evaluator is result.evaluator + assert wrapped._agent_name == "my-agent" + assert wrapped._runtime_id == "run-123" + finally: + result.dispose() + + async def test_returns_none_when_dispatcher_init_fails( + self, + monkeypatch: pytest.MonkeyPatch, + cwd: Path, + ) -> None: + """A dispatcher constructor blow-up must be swallowed — governance + is optional and a failing bootstrap must not crash the CLI. No + ``atexit`` hook should leak. + """ + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.is_governance_enabled", + lambda: True, + ) + _install_fake_runtime_governance( + monkeypatch, + audit_manager_cls=_FakeAuditManager, + metadata_cls=_FakeMetadata, + evaluator_cls=_FakeEvaluator, + compensator_cls=_FakeCompensator, + ) + _stub_provider( + monkeypatch, + response_or_exc=_fake_policy_response( + mode=EnforcementMode.ENFORCE, policies="rules: []" + ), + ) + + def _boom(_provider: Any) -> Any: + raise RuntimeError("dispatcher init exploded") + + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.LiveTrackEventDispatcher", + _boom, + ) + + registered: list[Any] = [] + + def _fake_register(func: Any, *_a: Any, **_kw: Any) -> Any: + registered.append(func) + return func + + def _fake_unregister(func: Any) -> None: + if func in registered: + registered.remove(func) + + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.atexit.register", _fake_register + ) + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.atexit.unregister", _fake_unregister + ) + + result = await resolve_governance( + agent_framework="langgraph", + agent_type="uipath_coded", + is_conversational=False, + ) + assert result is None + assert not registered, "atexit hook leaked when dispatcher init failed" + + async def test_returns_none_and_cleans_up_when_evaluator_setup_fails( + self, + monkeypatch: pytest.MonkeyPatch, + cwd: Path, + ) -> None: + """If a component built AFTER the dispatcher (e.g., the evaluator) + raises, ``resolve_governance`` must unregister the ``atexit`` hook + AND shut the dispatcher down before returning ``None``. + """ + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.is_governance_enabled", + lambda: True, + ) + + class _ExplodingEvaluator: + def __init__(self, *_a: Any, **_kw: Any) -> None: + raise RuntimeError("evaluator init exploded") + + _install_fake_runtime_governance( + monkeypatch, + audit_manager_cls=_FakeAuditManager, + metadata_cls=_FakeMetadata, + evaluator_cls=_ExplodingEvaluator, + compensator_cls=_FakeCompensator, + ) + _stub_provider( + monkeypatch, + response_or_exc=_fake_policy_response( + mode=EnforcementMode.ENFORCE, policies="rules: []" + ), + ) + + shutdown_calls: list[int] = [] + + class _FakeDispatcher: + def __init__(self, _provider: Any) -> None: + self.dispatch = MagicMock() + + def shutdown(self) -> None: + shutdown_calls.append(1) + + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.LiveTrackEventDispatcher", + _FakeDispatcher, + ) + + registered: list[Any] = [] + + def _fake_register(func: Any, *_a: Any, **_kw: Any) -> Any: + registered.append(func) + return func + + def _fake_unregister(func: Any) -> None: + if func in registered: + registered.remove(func) + + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.atexit.register", _fake_register + ) + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.atexit.unregister", _fake_unregister + ) + + result = await resolve_governance( + agent_framework="langgraph", + agent_type="uipath_coded", + is_conversational=False, + ) + assert result is None + assert not registered, "atexit hook not unregistered after evaluator failure" + assert shutdown_calls == [1], "dispatcher not shut down after evaluator failure" + + async def test_dispose_swallows_dispatcher_shutdown_errors( + self, + monkeypatch: pytest.MonkeyPatch, + cwd: Path, + ) -> None: + """``dispose`` runs from CLI ``finally`` blocks — it must never + raise, or it will mask the primary exception. A shutdown that + blows up should be logged at debug and swallowed. + """ + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.is_governance_enabled", + lambda: True, + ) + _install_fake_runtime_governance( + monkeypatch, + audit_manager_cls=_FakeAuditManager, + metadata_cls=_FakeMetadata, + evaluator_cls=_FakeEvaluator, + compensator_cls=_FakeCompensator, + ) + _stub_provider( + monkeypatch, + response_or_exc=_fake_policy_response( + mode=EnforcementMode.ENFORCE, policies="rules: []" + ), + ) + + class _ExplodingDispatcher: + def __init__(self, _provider: Any) -> None: + self.dispatch = MagicMock() + + def shutdown(self) -> None: + raise RuntimeError("shutdown exploded") + + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.LiveTrackEventDispatcher", + _ExplodingDispatcher, + ) + # atexit stubs so the exploding shutdown is never left registered + # against the real atexit — otherwise it would fire at pytest exit. + registered: list[Any] = [] + + def _fake_register(func: Any, *_a: Any, **_kw: Any) -> Any: + registered.append(func) + return func + + def _fake_unregister(func: Any) -> None: + if func in registered: + registered.remove(func) + + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.atexit.register", _fake_register + ) + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.atexit.unregister", _fake_unregister + ) + + result = await resolve_governance( + agent_framework="langgraph", + agent_type="uipath_coded", + is_conversational=False, + ) + assert result is not None + # Must not raise even though the dispatcher's shutdown does. + result.dispose() + # And must remain idempotent-safe. + result.dispose() + + async def test_dispose_atexit_hook_matches_dispatcher_shutdown( + self, + monkeypatch: pytest.MonkeyPatch, + cwd: Path, + ) -> None: + """The atexit hook that ``dispose`` unregisters must be the same + callable that ``atexit.register`` received — otherwise unregister + is a silent no-op and the dispatcher lingers. + """ + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.is_governance_enabled", + lambda: True, + ) + _install_fake_runtime_governance( + monkeypatch, + audit_manager_cls=_FakeAuditManager, + metadata_cls=_FakeMetadata, + evaluator_cls=_FakeEvaluator, + compensator_cls=_FakeCompensator, + ) + _stub_provider( + monkeypatch, + response_or_exc=_fake_policy_response( + mode=EnforcementMode.ENFORCE, policies="rules: []" + ), + ) + + registered_arg: list[Any] = [] + unregistered_arg: list[Any] = [] + + def _capture_register(func: Any) -> Any: + registered_arg.append(func) + return func + + def _capture_unregister(func: Any) -> None: + unregistered_arg.append(func) + + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.atexit.register", _capture_register + ) + monkeypatch.setattr( + "uipath._cli._governance_bootstrap.atexit.unregister", + _capture_unregister, + ) + + result = await resolve_governance( + agent_framework="langgraph", + agent_type="uipath_coded", + is_conversational=False, + ) + assert result is not None + + result.dispose() + + assert len(registered_arg) == 1 + assert len(unregistered_arg) == 1 + # Same bound method → same underlying dispatcher shutdown. + assert registered_arg[0] == unregistered_arg[0] diff --git a/packages/uipath/tests/cli/test_init.py b/packages/uipath/tests/cli/test_init.py index fa5b47ab2..b38725124 100644 --- a/packages/uipath/tests/cli/test_init.py +++ b/packages/uipath/tests/cli/test_init.py @@ -1,5 +1,7 @@ import json import os +import uuid +from typing import Any from unittest.mock import patch import pytest @@ -57,6 +59,101 @@ def test_init_creates_empty_uipath_json( assert isinstance(config["functions"], dict) assert len(config["functions"]) == 0 + def test_init_mints_agent_id_in_uipath_json( + self, runner: CliRunner, temp_dir: str + ) -> None: + """init writes a valid id into a newly created uipath.json.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + self._generate_pyproject() + result = runner.invoke(cli, ["init"], env={}) + assert result.exit_code == 0 + + with open("uipath.json", "r") as f: + config = json.load(f) + assert "id" in config + # Must be a valid UUID-shaped identifier. + uuid.UUID(config["id"]) + + def test_init_does_not_create_telemetry_file( + self, runner: CliRunner, temp_dir: str + ) -> None: + """init no longer writes .uipath/.telemetry.json; the id lives in uipath.json.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + self._generate_pyproject() + result = runner.invoke(cli, ["init"], env={}) + assert result.exit_code == 0 + + assert not os.path.exists(os.path.join(".uipath", ".telemetry.json")) + with open("uipath.json", "r") as f: + uuid.UUID(json.load(f)["id"]) + + def test_init_mints_agent_id_with_telemetry_disabled( + self, runner: CliRunner, temp_dir: str + ) -> None: + """id is still written when telemetry is opted out.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + self._generate_pyproject() + result = runner.invoke( + cli, ["init"], env={"UIPATH_TELEMETRY_ENABLED": "false"} + ) + assert result.exit_code == 0 + + assert not os.path.exists(os.path.join(".uipath", ".telemetry.json")) + with open("uipath.json", "r") as f: + config = json.load(f) + uuid.UUID(config["id"]) + + def test_init_preserves_existing_agent_id( + self, runner: CliRunner, temp_dir: str + ) -> None: + """init keeps an id already present in uipath.json (first writer wins).""" + with runner.isolated_filesystem(temp_dir=temp_dir): + with open("main.py", "w") as f: + f.write("def main(input: str) -> str: return input") + with open("uipath.json", "w") as f: + json.dump( + { + "id": "existing-agent-id", + "functions": {"main": "main.py:main"}, + }, + f, + ) + self._generate_pyproject() + + result = runner.invoke(cli, ["init"], env={}) + assert result.exit_code == 0 + # Existing id must not be backfilled/overwritten. + assert "with 'id'" not in result.output + + with open("uipath.json", "r") as f: + assert json.load(f)["id"] == "existing-agent-id" + + def test_init_backfills_agent_id_from_telemetry( + self, runner: CliRunner, temp_dir: str + ) -> None: + """init backfills id on an existing uipath.json, reusing the telemetry key.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + with open("main.py", "w") as f: + f.write("def main(input: str) -> str: return input") + with open("uipath.json", "w") as f: + json.dump({"functions": {"main": "main.py:main"}}, f) + os.makedirs(".uipath", exist_ok=True) + with open(os.path.join(".uipath", ".telemetry.json"), "w") as f: + json.dump({"ProjectKey": "legacy-project-key"}, f) + self._generate_pyproject() + + result = runner.invoke(cli, ["init"], env={}) + assert result.exit_code == 0 + assert "Updated 'uipath.json' file with 'id'" in result.output + + with open("uipath.json", "r") as f: + config = json.load(f) + assert config["id"] == "legacy-project-key" + # Existing fields are preserved. + assert config["functions"]["main"] == "main.py:main" + # The backfill is targeted: no defaulted fields are materialized. + assert set(config.keys()) == {"functions", "id"} + def test_init_with_existing_uipath_json( self, runner: CliRunner, temp_dir: str ) -> None: @@ -618,6 +715,94 @@ def test_init_mixed_entrypoints_warns( "We recommend using a single type for all entrypoints" in result.output ) + @pytest.mark.parametrize("entrypoint_kind", ["functions", "agents"]) + def test_init_vertical_solution_stamps_transaction_root( + self, runner: CliRunner, temp_dir: str, entrypoint_kind: str + ) -> None: + """'_uipathVerticalSolution: true' stamps isTransactionRoot on entrypoints.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + with open("main.py", "w") as f: + f.write("def main(input: str) -> str: return input") + with open("uipath.json", "w") as f: + json.dump( + { + "runtimeOptions": {"_uipathVerticalSolution": True}, + entrypoint_kind: {"main": "main.py:main"}, + }, + f, + ) + self._generate_pyproject() + + result = runner.invoke(cli, ["init"], env={}) + assert result.exit_code == 0 + + with open("entry-points.json", "r") as f: + entry_points = json.load(f)["entryPoints"] + assert len(entry_points) == 1 + assert entry_points[0]["isTransactionRoot"] is True + + @pytest.mark.parametrize( + "runtime_options", + [None, {}, {"_uipathVerticalSolution": False}], + ) + def test_init_without_vertical_solution_omits_transaction_root( + self, runner: CliRunner, temp_dir: str, runtime_options: dict[str, Any] | None + ) -> None: + """Entrypoints are not stamped when the flag is absent or false.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + with open("main.py", "w") as f: + f.write("def main(input: str) -> str: return input") + uipath_config: dict[str, Any] = {"functions": {"main": "main.py:main"}} + if runtime_options is not None: + uipath_config["runtimeOptions"] = runtime_options + with open("uipath.json", "w") as f: + json.dump(uipath_config, f) + self._generate_pyproject() + + result = runner.invoke(cli, ["init"], env={}) + assert result.exit_code == 0 + + with open("entry-points.json", "r") as f: + entry_points = json.load(f)["entryPoints"] + assert "isTransactionRoot" not in entry_points[0] + + def test_init_rerun_syncs_transaction_root_with_vertical_solution_flag( + self, runner: CliRunner, temp_dir: str + ) -> None: + """Rerunning init keeps or removes isTransactionRoot to match the flag.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + with open("main.py", "w") as f: + f.write("def main(input: str) -> str: return input") + with open("uipath.json", "w") as f: + json.dump( + { + "runtimeOptions": {"_uipathVerticalSolution": True}, + "functions": {"main": "main.py:main"}, + }, + f, + ) + self._generate_pyproject() + + assert runner.invoke(cli, ["init"], env={}).exit_code == 0 + with open("entry-points.json", "r") as f: + assert json.load(f)["entryPoints"][0]["isTransactionRoot"] is True + + # Rerun with the flag still present: the stamp is kept. + assert runner.invoke(cli, ["init"], env={}).exit_code == 0 + with open("entry-points.json", "r") as f: + assert json.load(f)["entryPoints"][0]["isTransactionRoot"] is True + + # Remove the flag (keep the backfilled id and functions untouched). + with open("uipath.json", "r") as f: + uipath_config = json.load(f) + del uipath_config["runtimeOptions"]["_uipathVerticalSolution"] + with open("uipath.json", "w") as f: + json.dump(uipath_config, f) + + assert runner.invoke(cli, ["init"], env={}).exit_code == 0 + with open("entry-points.json", "r") as f: + assert "isTransactionRoot" not in json.load(f)["entryPoints"][0] + def test_init_creates_studio_metadata_file( self, runner: CliRunner, temp_dir: str ) -> None: @@ -659,3 +844,29 @@ def test_init_does_not_overwrite_existing_studio_metadata( metadata = json.load(f) assert metadata["schemaVersion"] == 99 assert metadata["codeVersion"] == "5.0.0" + + +class TestWriteMermaidFiles: + def test_mermaid_file_starts_with_header_comment( + self, runner: CliRunner, temp_dir: str + ) -> None: + """Generated .mermaid files begin with the clarifying header comment.""" + from uipath._cli.cli_init import MERMAID_FILE_HEADER, write_mermaid_files + from uipath.runtime.schema import UiPathRuntimeGraph, UiPathRuntimeSchema + + ep = UiPathRuntimeSchema( + filePath="main.py", + uniqueId="main", + type="function", + input={}, + output={}, + graph=UiPathRuntimeGraph(), + ) + + with runner.isolated_filesystem(temp_dir=temp_dir): + paths = write_mermaid_files([ep]) + assert len(paths) == 1 + contents = paths[0].read_text() + assert contents.startswith(MERMAID_FILE_HEADER) + assert "AUTO-GENERATED" in contents + assert "uipath init" in contents diff --git a/packages/uipath/tests/cli/test_pack.py b/packages/uipath/tests/cli/test_pack.py index cf6bd5cc1..cfb1f1a0f 100644 --- a/packages/uipath/tests/cli/test_pack.py +++ b/packages/uipath/tests/cli/test_pack.py @@ -1,26 +1,28 @@ # type: ignore +import io import json import os import zipfile from unittest.mock import patch +import pytest from click.testing import CliRunner from utils.project_details import ProjectDetails import uipath._cli.cli_pack as cli_pack from uipath._cli import cli from uipath._cli.middlewares import MiddlewareResult -from uipath._cli.models.uipath_json_schema import RuntimeOptions +from uipath._cli.models.uipath_json_schema import RuntimeOptions, UiPathJsonConfig -def create_bindings_file(): +def create_bindings_file(directory: str = "."): """Helper to create a default bindings.json file for tests.""" bindings_content = {"version": "2.0", "resources": []} - with open("bindings.json", "w") as f: + with open(os.path.join(directory, "bindings.json"), "w") as f: json.dump(bindings_content, f, indent=4) -def create_entry_points_file(entrypoint_type: str = "function"): +def create_entry_points_file(entrypoint_type: str = "function", directory: str = "."): """Helper to create a default entry-points.json file for tests.""" entry_points_content = { "$schema": "https://cloud.uipath.com/draft/2024-12/entry-point", @@ -38,7 +40,7 @@ def create_entry_points_file(entrypoint_type: str = "function"): } ], } - with open("entry-points.json", "w") as f: + with open(os.path.join(directory, "entry-points.json"), "w") as f: json.dump(entry_points_content, f, indent=4) @@ -328,6 +330,57 @@ def test_include_file_extensions( "Binary file content was corrupted during packing" ) + def test_include_wheel_file_not_corrupted( + self, + runner: CliRunner, + temp_dir: str, + project_details: ProjectDetails, + ) -> None: + """Test that .whl files included via packOptions are packed byte-for-byte. + + A .whl file is itself a zip archive full of arbitrary binary bytes. If it + is not recognized as binary by the packager, it gets round-tripped through + a text decode/encode (latin-1 -> UTF-8), which corrupts any byte >= 0x80 + and produces an invalid zip file at runtime. + """ + wheel_file_name = "example_pkg-1.0.0-py3-none-any.whl" + + # Minimal valid zip (wheel) content, deliberately containing high bytes + # (0x80-0xff) that would be mangled by a latin-1 -> UTF-8 round trip. + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as wheel_zip: + wheel_zip.writestr("example_pkg/__init__.py", bytes(range(256)) * 4) + wheel_bytes = buf.getvalue() + + pack_options = {"fileExtensionsIncluded": [".whl"]} + + with runner.isolated_filesystem(temp_dir=temp_dir): + with open("uipath.json", "w") as f: + json.dump(create_uipath_json(pack_options=pack_options), f) + with open("pyproject.toml", "w") as f: + f.write(project_details.to_toml()) + with open("main.py", "w") as f: + f.write("def main(input): return input") + with open(wheel_file_name, "wb") as f: + f.write(wheel_bytes) + + with patch("uipath._cli.cli_init.Middlewares.next") as mock_middleware: + mock_middleware.return_value = MiddlewareResult(should_continue=True) + init_result = runner.invoke(cli, ["init"], env={}) + assert init_result.exit_code == 0 + + result = runner.invoke(cli, ["pack", "./"], env={}) + + assert result.exit_code == 0 + with zipfile.ZipFile( + f".uipath/{project_details.name}.{project_details.version}.nupkg", "r" + ) as z: + assert f"content/{wheel_file_name}" in z.namelist() + extracted_wheel_bytes = z.read(f"content/{wheel_file_name}") + assert extracted_wheel_bytes == wheel_bytes, ( + ".whl file content was corrupted during packing" + ) + def test_include_files( self, runner: CliRunner, @@ -1096,14 +1149,17 @@ def test_generate_operate_file(self, runner: CliRunner, temp_dir: str) -> None: ) ] - operate_data = cli_pack.generate_operate_file( - entrypoints, RuntimeOptions(is_conversational=False) + config = UiPathJsonConfig( + runtimeOptions=RuntimeOptions(is_conversational=False), + id="00000000-0000-0000-0000-000000000001", ) + operate_data = cli_pack.generate_operate_file(entrypoints, config) assert ( operate_data["$schema"] == "https://cloud.uipath.com/draft/2024-12/entry-point" ) + assert operate_data["projectId"] == "00000000-0000-0000-0000-000000000001" assert operate_data["main"] == "agent1.py" assert operate_data["contentType"] == "agent" assert operate_data["targetFramework"] == "Portable" @@ -1114,6 +1170,119 @@ def test_generate_operate_file(self, runner: CliRunner, temp_dir: str) -> None: "isConversational": False, } + def test_pack_uses_agent_id_as_project_id( + self, + runner: CliRunner, + temp_dir: str, + project_details: ProjectDetails, + ) -> None: + """operate.json projectId is sourced from uipath.json#id.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + config = create_uipath_json() + config["id"] = "00000000-0000-0000-0000-000000000001" + with open("uipath.json", "w") as f: + json.dump(config, f) + with open("pyproject.toml", "w") as f: + f.write(project_details.to_toml()) + with open("main.py", "w") as f: + f.write("def main(input): return input") + create_bindings_file() + create_entry_points_file() + + result = runner.invoke(cli, ["pack", "./"], env={}) + assert result.exit_code == 0 + + with zipfile.ZipFile( + f".uipath/{project_details.name}.{project_details.version}.nupkg", "r" + ) as z: + operate_data = json.loads(z.read("content/operate.json")) + assert operate_data["projectId"] == "00000000-0000-0000-0000-000000000001" + + def test_pack_fails_when_id_is_not_a_guid( + self, + runner: CliRunner, + temp_dir: str, + project_details: ProjectDetails, + ) -> None: + """pack fails when uipath.json#id is set but is not a valid GUID.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + config = create_uipath_json() + config["id"] = "not-a-guid" + with open("uipath.json", "w") as f: + json.dump(config, f) + with open("pyproject.toml", "w") as f: + f.write(project_details.to_toml()) + with open("main.py", "w") as f: + f.write("def main(input): return input") + create_bindings_file() + create_entry_points_file() + + result = runner.invoke(cli, ["pack", "./"], env={}) + assert result.exit_code != 0 + assert "must be a valid GUID" in result.output + + def test_pack_falls_back_to_telemetry_project_key( + self, + runner: CliRunner, + temp_dir: str, + project_details: ProjectDetails, + ) -> None: + """Without id, operate.json projectId falls back to the telemetry key.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + # uipath.json deliberately has no id (legacy project). + with open("uipath.json", "w") as f: + json.dump(create_uipath_json(), f) + with open("pyproject.toml", "w") as f: + f.write(project_details.to_toml()) + with open("main.py", "w") as f: + f.write("def main(input): return input") + create_bindings_file() + create_entry_points_file() + os.makedirs(".uipath", exist_ok=True) + with open(os.path.join(".uipath", ".telemetry.json"), "w") as f: + json.dump({"ProjectKey": "telemetry-fallback-key"}, f) + + result = runner.invoke(cli, ["pack", "./"], env={}) + assert result.exit_code == 0 + + with zipfile.ZipFile( + f".uipath/{project_details.name}.{project_details.version}.nupkg", "r" + ) as z: + operate_data = json.loads(z.read("content/operate.json")) + assert operate_data["projectId"] == "telemetry-fallback-key" + + def test_pack_telemetry_fallback_from_outside_project_dir( + self, + runner: CliRunner, + temp_dir: str, + project_details: ProjectDetails, + ) -> None: + """The legacy telemetry fallback resolves against the packed directory, not CWD.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + os.makedirs("project") + with open(os.path.join("project", "uipath.json"), "w") as f: + json.dump(create_uipath_json(), f) + with open(os.path.join("project", "pyproject.toml"), "w") as f: + f.write(project_details.to_toml()) + with open(os.path.join("project", "main.py"), "w") as f: + f.write("def main(input): return input") + create_bindings_file(directory="project") + create_entry_points_file(directory="project") + os.makedirs(os.path.join("project", ".uipath"), exist_ok=True) + with open(os.path.join("project", ".uipath", ".telemetry.json"), "w") as f: + json.dump({"ProjectKey": "telemetry-fallback-key"}, f) + + result = runner.invoke(cli, ["pack", "./project"], env={}) + assert result.exit_code == 0 + + # the package itself is written under the caller's CWD + with zipfile.ZipFile( + f".uipath/{project_details.name}.{project_details.version}.nupkg", + "r", + ) as z: + operate_data = json.loads(z.read("content/operate.json")) + assert operate_data["projectId"] == "telemetry-fallback-key" + def test_generate_bindings_content(self, runner: CliRunner, temp_dir: str) -> None: """Test generating bindings content.""" bindings_data = cli_pack.generate_bindings_content() @@ -1292,3 +1461,177 @@ def test_pack_warns_mixed_entrypoint_types( assert ( "We recommend using a single type for all entrypoints" in result.output ) + + +class TestPackMetadataConflicts: + """Test that project files cannot shadow generated package metadata.""" + + def _setup_project(self, project_details: ProjectDetails, pack_options=None): + with open("uipath.json", "w") as f: + json.dump(create_uipath_json(pack_options=pack_options), f) + with open("pyproject.toml", "w") as f: + f.write(project_details.to_toml()) + with open("main.py", "w") as f: + f.write("def main(input): return input") + create_bindings_file() + create_entry_points_file() + + @pytest.mark.parametrize( + "conflicting_file", + ["operate.json", "package-descriptor.json", "bindings_v2.json"], + ) + def test_pack_fails_when_metadata_file_exists_in_project( + self, + runner: CliRunner, + temp_dir: str, + project_details: ProjectDetails, + conflicting_file: str, + ) -> None: + """Test that a project file named like generated metadata blocks packing.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + self._setup_project(project_details) + with open(conflicting_file, "w") as f: + json.dump({"stale": True}, f) + + result = runner.invoke(cli, ["pack", "./"], env={}) + + assert result.exit_code == 1 + assert "These project files clash with generated package metadata:" in ( + result.output + ) + assert f"- {conflicting_file}" in result.output + assert "packOptions.filesExcluded" in result.output + assert "specs/uipath.spec.md#4-packoptions" in result.output + assert not os.path.exists( + f".uipath/{project_details.name}.{project_details.version}.nupkg" + ) + + def test_pack_reports_all_conflicting_files( + self, + runner: CliRunner, + temp_dir: str, + project_details: ProjectDetails, + ) -> None: + """Test that every conflicting file is listed, not just the first one.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + self._setup_project(project_details) + for conflicting_file in ("operate.json", "package-descriptor.json"): + with open(conflicting_file, "w") as f: + json.dump({}, f) + + result = runner.invoke(cli, ["pack", "./"], env={}) + + assert result.exit_code == 1 + assert "- operate.json" in result.output + assert "- package-descriptor.json" in result.output + + def test_pack_conflict_detection_is_case_insensitive( + self, + runner: CliRunner, + temp_dir: str, + project_details: ProjectDetails, + ) -> None: + """Test that a case variant is rejected, since extraction is case-insensitive.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + self._setup_project(project_details) + with open("Operate.json", "w") as f: + json.dump({}, f) + + result = runner.invoke(cli, ["pack", "./"], env={}) + + assert result.exit_code == 1 + assert "- Operate.json" in result.output + + def test_pack_allows_metadata_names_in_subdirectories( + self, + runner: CliRunner, + temp_dir: str, + project_details: ProjectDetails, + ) -> None: + """Test that the same file name below the project root is not a conflict.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + self._setup_project(project_details) + os.makedirs("fixtures") + with open(os.path.join("fixtures", "operate.json"), "w") as f: + json.dump({"fixture": True}, f) + + result = runner.invoke(cli, ["pack", "./"], env={}) + + assert result.exit_code == 0 + nupkg_path = ( + f".uipath/{project_details.name}.{project_details.version}.nupkg" + ) + with zipfile.ZipFile(nupkg_path, "r") as z: + names = z.namelist() + assert "content/fixtures/operate.json" in names + assert names.count("content/operate.json") == 1 + + def test_pack_succeeds_when_conflicting_file_is_excluded( + self, + runner: CliRunner, + temp_dir: str, + project_details: ProjectDetails, + ) -> None: + """Test that packOptions.filesExcluded resolves the conflict.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + self._setup_project( + project_details, pack_options={"filesExcluded": ["operate.json"]} + ) + with open("operate.json", "w") as f: + json.dump({"stale": True}, f) + + result = runner.invoke(cli, ["pack", "./"], env={}) + + assert result.exit_code == 0 + nupkg_path = ( + f".uipath/{project_details.name}.{project_details.version}.nupkg" + ) + with zipfile.ZipFile(nupkg_path, "r") as z: + names = z.namelist() + assert names.count("content/operate.json") == 1 + assert json.loads(z.read("content/operate.json")) != {"stale": True} + + def test_nupkg_has_no_duplicate_entries( + self, + runner: CliRunner, + temp_dir: str, + project_details: ProjectDetails, + ) -> None: + """Test that a packed project never contains duplicate archive entries.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + self._setup_project(project_details) + + result = runner.invoke(cli, ["pack", "./"], env={}) + assert result.exit_code == 0 + + nupkg_path = ( + f".uipath/{project_details.name}.{project_details.version}.nupkg" + ) + with zipfile.ZipFile(nupkg_path, "r") as z: + names = z.namelist() + assert len(names) == len(set(names)) + + def test_pack_without_bindings_file( + self, + runner: CliRunner, + temp_dir: str, + project_details: ProjectDetails, + ) -> None: + """Test that packing works when bindings.json is absent.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + with open("uipath.json", "w") as f: + json.dump(create_uipath_json(), f) + with open("pyproject.toml", "w") as f: + f.write(project_details.to_toml()) + with open("main.py", "w") as f: + f.write("def main(input): return input") + create_entry_points_file() + + result = runner.invoke(cli, ["pack", "./"], env={}) + + assert result.exit_code == 0, result.output + nupkg_path = ( + f".uipath/{project_details.name}.{project_details.version}.nupkg" + ) + with zipfile.ZipFile(nupkg_path, "r") as z: + assert "content/bindings_v2.json" not in z.namelist() diff --git a/packages/uipath/tests/cli/test_push.py b/packages/uipath/tests/cli/test_push.py index d2189098d..a9af55f14 100644 --- a/packages/uipath/tests/cli/test_push.py +++ b/packages/uipath/tests/cli/test_push.py @@ -712,6 +712,80 @@ def test_push_non_coded_agent_project( in result.output ) + def test_first_push_to_uninitialized_project( + self, + runner: CliRunner, + temp_dir: str, + project_details: ProjectDetails, + mock_env_vars: dict[str, str], + httpx_mock: HTTPXMock, + ) -> None: + """Test push when the remote file system was never initialized. + + The backend returns 404 from FileOperations/Structure for projects + whose file system does not exist yet (e.g. a freshly created Function + project). The first push should bootstrap the files instead of failing + the coded-agent validation. + """ + base_url = "https://cloud.uipath.com/organization" + project_id = "test-project-id" + + # Uninitialized file system: Structure returns 404 + httpx_mock.add_response( + url=f"{base_url}/studio_/backend/api/Project/{project_id}/FileOperations/Structure", + status_code=404, + json={ + "type": "https://tools.ietf.org/html/rfc9110#section-15.5.5", + "title": "Not Found", + "status": 404, + }, + ) + + self._mock_lock_retrieval(httpx_mock, base_url, project_id, times=1) + + httpx_mock.add_response( + method="POST", + url=f"{base_url}/studio_/backend/api/Project/{project_id}/FileOperations/StructuralMigration", + status_code=200, + json={"success": True}, + ) + + # Empty folder cleanup - get structure again after migration + httpx_mock.add_response( + url=f"{base_url}/studio_/backend/api/Project/{project_id}/FileOperations/Structure", + json={ + "id": "root", + "name": "root", + "folders": [], + "files": [], + "folderType": "0", + }, + ) + + with runner.isolated_filesystem(temp_dir=temp_dir): + self._create_required_files() + + with open("pyproject.toml", "w") as f: + f.write(project_details.to_toml()) + + with open("main.py", "w") as f: + f.write("print('Hello World')") + + with open("uv.lock", "w") as f: + f.write('version = 1 \n requires-python = ">=3.11"') + + configure_env_vars(mock_env_vars) + os.environ["UIPATH_PROJECT_ID"] = project_id + + result = runner.invoke(cli, ["push", "./", "--ignore-resources"]) + assert result.exit_code == 0 + assert "not of type coded agent" not in result.output + assert "Uploading 'main.py'" in result.output + assert "Uploading 'pyproject.toml'" in result.output + assert "Uploading 'uipath.json'" in result.output + assert "Uploading 'uv.lock'" in result.output + assert "Uploading '.uipath/studio_metadata.json'" in result.output + def test_push_with_nolock_flag( self, runner: CliRunner, @@ -1928,6 +2002,14 @@ def test_push_with_resources_imports_referenced_resources( json=mock_structure, ) + httpx_mock.add_response( + method="GET", + url=f"{base_url}/studio_/backend/api/resourcebuilder/metadata", + json=[ + {"kind": "asset", "versions": [{"supportsInLineCreation": True}]}, + ], + ) + # Mock getting the solution ID httpx_mock.add_response( method="GET", @@ -2171,7 +2253,7 @@ def test_push_with_ignore_resources_flag_skips_resource_import( assert "Created reference for resource" not in result.output assert "Resource import summary" not in result.output - def test_push_with_resource_not_found_shows_warning( + def test_push_with_resource_not_found_creates_virtual( self, runner: CliRunner, temp_dir: str, @@ -2179,9 +2261,11 @@ def test_push_with_resource_not_found_shows_warning( mock_env_vars: dict[str, str], httpx_mock: HTTPXMock, ) -> None: - """Test that push shows warning when referenced resource is not found in catalog.""" + """When catalog lookup misses, push creates a virtual resource placeholder.""" base_url = "https://cloud.uipath.com/organization" project_id = "test-project-id" + solution_id = "test-solution-id" + tenant_id = "test-tenant-id" mock_structure = { "id": "root", @@ -2226,6 +2310,43 @@ def test_push_with_resource_not_found_shows_warning( json=mock_structure, ) + # Resource Builder metadata — declares which kinds support inline creation. + httpx_mock.add_response( + method="GET", + url=f"{base_url}/studio_/backend/api/resourcebuilder/metadata", + json=[ + {"kind": "asset", "versions": [{"supportsInLineCreation": True}]}, + ], + ) + + # Solution ID lookup for the virtual-resource fallback + httpx_mock.add_response( + method="GET", + url=f"{base_url}/studio_/backend/api/Project/{project_id}", + json={"solutionId": solution_id}, + ) + + # Existing resources in the solution (empty → no conflict with our new virtual) + httpx_mock.add_response( + method="GET", + url=f"{base_url}/studio_/backend/api/resourcebuilder/solutions/{solution_id}/entities", + json={"resources": []}, + ) + + # Virtual resource POST + httpx_mock.add_response( + method="POST", + url=f"{base_url}/studio_/backend/api/resourcebuilder/solutions/{solution_id}/resources/virtual", + json={"key": "virtual-resource-key-123"}, + ) + + # Configuration PATCH after virtual creation + httpx_mock.add_response( + method="PATCH", + url=f"{base_url}/studio_/backend/api/resourcebuilder/solutions/{solution_id}/resources/virtual-resource-key-123/configuration", + json={}, + ) + with runner.isolated_filesystem(temp_dir=temp_dir): # Create required files with open("uipath.json", "w") as f: @@ -2255,6 +2376,7 @@ def test_push_with_resource_not_found_shows_warning( "ActivityName": "retrieve_async", "BindingsVersion": "2.2", "DisplayLabel": "FullName", + "SubType": "stringAsset", }, } ], @@ -2273,6 +2395,7 @@ def test_push_with_resource_not_found_shows_warning( configure_env_vars(mock_env_vars) os.environ["UIPATH_PROJECT_ID"] = project_id + os.environ["UIPATH_TENANT_ID"] = tenant_id # Mock resource catalog list_by_type_async to return no resources async def mock_list_by_type_async_empty(*args, **kwargs): @@ -2291,16 +2414,14 @@ async def mock_list_by_type_async_empty(*args, **kwargs): result = runner.invoke(cli, ["push", "./"]) assert result.exit_code == 0 - # Check that warning was shown for missing resource + # Check that the virtual-resource fallback ran and succeeded assert ( "Importing referenced resources to Studio Web project" in result.output ) - assert ( - "Resource 'missing.asset' of type 'asset' at folder path 'Default' was not found" - in result.output - ) + assert "missing.asset" in result.output + assert "created successfully" in result.output assert "Resource import summary:" in result.output - assert "1 not found" in result.output + assert "1 virtual-created" in result.output def test_push_with_resource_already_exists_shows_unchanged( self, @@ -2359,6 +2480,14 @@ def test_push_with_resource_already_exists_shows_unchanged( json=mock_structure, ) + httpx_mock.add_response( + method="GET", + url=f"{base_url}/studio_/backend/api/resourcebuilder/metadata", + json=[ + {"kind": "asset", "versions": [{"supportsInLineCreation": True}]}, + ], + ) + # Mock getting the solution ID httpx_mock.add_response( method="GET", diff --git a/packages/uipath/tests/cli/test_run.py b/packages/uipath/tests/cli/test_run.py index 9069c5426..aa182c7c5 100644 --- a/packages/uipath/tests/cli/test_run.py +++ b/packages/uipath/tests/cli/test_run.py @@ -1,6 +1,8 @@ # type: ignore +import json import os -from unittest.mock import patch +from contextlib import asynccontextmanager +from unittest.mock import AsyncMock, Mock, patch import pytest from click.testing import CliRunner @@ -9,6 +11,41 @@ from uipath._cli.middlewares import MiddlewareResult +def _middleware_continue(): + return MiddlewareResult( + should_continue=True, + error_message=None, + should_include_stacktrace=False, + ) + + +async def _empty_async_gen(*args, **kwargs): + """An async generator that yields nothing (simulates empty runtime.stream).""" + if False: # pragma: no cover + yield + + +def _make_mock_factory(entrypoints: list[str]): + """Create a mock runtime factory with given entrypoints.""" + mock_factory = Mock() + mock_factory.discover_entrypoints.return_value = entrypoints + mock_factory.get_settings = AsyncMock(return_value=None) + mock_factory.dispose = AsyncMock() + + mock_runtime = Mock() + mock_runtime.execute = AsyncMock(return_value=Mock(status="SUCCESSFUL")) + mock_runtime.stream = Mock(side_effect=_empty_async_gen) + mock_runtime.dispose = AsyncMock() + mock_factory.new_runtime = AsyncMock(return_value=mock_runtime) + + return mock_factory + + +@asynccontextmanager +async def _mock_resource_overwrites_context(*args, **kwargs): + yield + + @pytest.fixture def entrypoint(): return "main" @@ -142,14 +179,81 @@ def test_run_input_file_success( assert "Successful execution." in result.output class TestMiddleware: - def test_no_entrypoint(self, runner: CliRunner, temp_dir: str): + def test_autodiscover_entrypoint(self, runner: CliRunner, temp_dir: str): + """When exactly one entrypoint exists, it is auto-resolved.""" with runner.isolated_filesystem(temp_dir=temp_dir): - result = runner.invoke(cli, ["run"]) - assert result.exit_code == 1 - assert ( - "No entrypoint specified" in result.output - or "Missing argument" in result.output + mock_factory = _make_mock_factory(["my_agent"]) + + with ( + patch( + "uipath._cli.cli_run.Middlewares.next", + return_value=_middleware_continue(), + ), + patch( + "uipath._cli.cli_run.UiPathRuntimeFactoryRegistry.get", + return_value=mock_factory, + ), + patch( + "uipath._cli.cli_run.ResourceOverwritesContext", + side_effect=_mock_resource_overwrites_context, + ), + ): + result = runner.invoke(cli, ["run"]) + + assert result.exit_code == 0, ( + f"output: {result.output!r}, exception: {result.exception}" ) + assert "Successful execution." in result.output + mock_factory.new_runtime.assert_awaited_once() + assert mock_factory.new_runtime.call_args[0][0] == "my_agent" + + def test_no_entrypoint_multiple_available( + self, runner: CliRunner, temp_dir: str + ): + """When multiple entrypoints exist and none specified, show usage help.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + mock_factory = _make_mock_factory(["agent_a", "agent_b"]) + + with ( + patch( + "uipath._cli.cli_run.Middlewares.next", + return_value=_middleware_continue(), + ), + patch( + "uipath._cli.cli_run.UiPathRuntimeFactoryRegistry.get", + return_value=mock_factory, + ), + ): + result = runner.invoke(cli, ["run"]) + + assert result.exit_code == 0 + assert "Available entrypoints:" in result.output + assert "agent_a" in result.output + assert "agent_b" in result.output + assert "Usage: uipath run" in result.output + mock_factory.new_runtime.assert_not_awaited() + + def test_no_entrypoint_none_available(self, runner: CliRunner, temp_dir: str): + """When no entrypoints exist and none specified, show usage help.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + mock_factory = _make_mock_factory([]) + + with ( + patch( + "uipath._cli.cli_run.Middlewares.next", + return_value=_middleware_continue(), + ), + patch( + "uipath._cli.cli_run.UiPathRuntimeFactoryRegistry.get", + return_value=mock_factory, + ), + ): + result = runner.invoke(cli, ["run"]) + + assert result.exit_code == 0 + assert "No entrypoints found" in result.output + assert "Usage: uipath run" in result.output + mock_factory.new_runtime.assert_not_awaited() def test_script_not_found( self, runner: CliRunner, temp_dir: str, entrypoint: str @@ -345,3 +449,113 @@ def main(input_data: PersonIn) -> PersonOut: assert output_data["email"] == "john@example.com" assert output_data["is_adult"] is True assert output_data["greeting"] == "Hello, John Doe!" + + +_SIMULATION_JSON = { + "enabled": True, + "toolsToSimulate": [{"name": "check_syntax"}, {"name": "check_style"}], + "instructions": "Simulate.", +} + + +class TestRunSimulation: + """Tests for the --simulation flag on the run command.""" + + def _make_factory(self): + factory = Mock() + runtime = Mock() + runtime.stream = Mock(side_effect=_empty_async_gen) + runtime.dispose = AsyncMock() + runtime.get_schema = AsyncMock(return_value=Mock(metadata=None)) + factory.discover_entrypoints.return_value = ["main"] + factory.get_settings = AsyncMock(return_value=None) + factory.dispose = AsyncMock() + factory.new_runtime = AsyncMock(return_value=runtime) + return factory, runtime + + def test_invalid_simulation_json_exits_with_error( + self, runner: CliRunner, temp_dir: str + ): + with runner.isolated_filesystem(temp_dir=temp_dir): + with open("uipath.json", "w") as f: + json.dump({"functions": {"main": "main.py:main"}}, f) + with open("main.py", "w") as f: + f.write("async def main(input): return {}") + + result = runner.invoke( + cli, ["run", "main", "--simulation", "{ not valid json }"] + ) + assert result.exit_code == 1 + assert "Invalid JSON" in result.output + + def test_simulation_wraps_runtime_with_mock_runtime( + self, runner: CliRunner, temp_dir: str + ): + factory, _ = self._make_factory() + + with runner.isolated_filesystem(temp_dir=temp_dir): + with open("uipath.json", "w") as f: + json.dump({"functions": {"main": "main.py:main"}}, f) + with open("main.py", "w") as f: + f.write("async def main(input): return {}") + + with ( + patch( + "uipath._cli.cli_run.Middlewares.next", + return_value=_middleware_continue(), + ), + patch( + "uipath._cli.cli_run.UiPathRuntimeFactoryRegistry.get", + return_value=factory, + ), + patch( + "uipath._cli.cli_run.ResourceOverwritesContext", + side_effect=_mock_resource_overwrites_context, + ), + patch("uipath._cli.cli_run.UiPathMockRuntime") as mock_cls, + ): + mock_cls.return_value = Mock( + stream=Mock(side_effect=_empty_async_gen), + dispose=AsyncMock(), + get_schema=AsyncMock(return_value=Mock(metadata=None)), + ) + runner.invoke( + cli, + ["run", "main", "--simulation", json.dumps(_SIMULATION_JSON)], + ) + + assert mock_cls.called + assert mock_cls.call_args.kwargs["mocking_context"] is not None + + def test_simulation_disabled_does_not_wrap_runtime( + self, runner: CliRunner, temp_dir: str + ): + factory, _ = self._make_factory() + disabled = {**_SIMULATION_JSON, "enabled": False} + + with runner.isolated_filesystem(temp_dir=temp_dir): + with open("uipath.json", "w") as f: + json.dump({"functions": {"main": "main.py:main"}}, f) + with open("main.py", "w") as f: + f.write("async def main(input): return {}") + + with ( + patch( + "uipath._cli.cli_run.Middlewares.next", + return_value=_middleware_continue(), + ), + patch( + "uipath._cli.cli_run.UiPathRuntimeFactoryRegistry.get", + return_value=factory, + ), + patch( + "uipath._cli.cli_run.ResourceOverwritesContext", + side_effect=_mock_resource_overwrites_context, + ), + patch("uipath._cli.cli_run.UiPathMockRuntime") as mock_cls, + ): + runner.invoke( + cli, ["run", "main", "--simulation", json.dumps(disabled)] + ) + + assert not mock_cls.called diff --git a/packages/uipath/tests/cli/test_server.py b/packages/uipath/tests/cli/test_server.py index 185979924..70d29cb38 100644 --- a/packages/uipath/tests/cli/test_server.py +++ b/packages/uipath/tests/cli/test_server.py @@ -1,14 +1,39 @@ import asyncio import json import os -import threading +import subprocess +import sys import time -from typing import Any +from urllib.request import urlopen import aiohttp import pytest -from uipath._cli.cli_server import start_tcp_server +from tests.cli.utils.server import ( + get_free_port, + start_cli_server_thread, + start_job, + start_job_with_env, +) + +SERVER_START_TIMEOUT_SECONDS = 20 +SERVER_STOP_TIMEOUT_SECONDS = 5 +JOB_START_TIMEOUT_SECONDS = 30 + + +@pytest.fixture +def simple_script() -> str: + return """ +from dataclasses import dataclass + +@dataclass +class Input: + message: str + repeat: int = 1 + +def main(input: Input) -> str: + return (input.message + " ") * input.repeat +""" def create_uipath_json(script_path: str, entrypoint_name: str = "main"): @@ -16,82 +41,64 @@ def create_uipath_json(script_path: str, entrypoint_name: str = "main"): return {"functions": {entrypoint_name: f"{script_path}:main"}} -async def start_job( - port: int, job_key: str, command: str, args: list[str] -) -> dict[str, Any]: - """Start a job on the server.""" - async with aiohttp.ClientSession() as session: - async with session.post( - f"http://127.0.0.1:{port}/jobs/{job_key}/start", - json={"command": command, "args": args}, - ) as response: - return await response.json() - - -async def start_job_with_env( - port: int, - job_key: str, - command: str, - args: list[str], - env_vars: dict[str, str], -) -> dict[str, Any]: - """Start a job on the server with environment variables.""" - async with aiohttp.ClientSession() as session: - async with session.post( - f"http://127.0.0.1:{port}/jobs/{job_key}/start", - json={ - "command": command, - "args": args, - "environmentVariables": env_vars, - }, - ) as response: - return await response.json() +def read_process_output(process: subprocess.Popen[str]) -> str: + if process.stdout is None: + return "" + return process.stdout.read() + + +def stop_process(process: subprocess.Popen[str]) -> str: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=SERVER_STOP_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=SERVER_STOP_TIMEOUT_SECONDS) + return read_process_output(process) + + +def wait_for_server_health(port: int, process: subprocess.Popen[str]) -> None: + deadline = time.monotonic() + SERVER_START_TIMEOUT_SECONDS + last_error: Exception | None = None + + while time.monotonic() < deadline: + if process.poll() is not None: + output = read_process_output(process) + raise AssertionError( + f"uipath server exited before becoming healthy with code " + f"{process.returncode}\n\n{output}" + ) + + try: + with urlopen(f"http://127.0.0.1:{port}/health", timeout=1) as response: + if response.status == 200 and response.read().decode() == "OK": + return + except Exception as e: + last_error = e + + time.sleep(0.1) + + output = stop_process(process) + raise AssertionError( + f"uipath server did not become healthy within " + f"{SERVER_START_TIMEOUT_SECONDS}s. Last error: {last_error}\n\n{output}" + ) class TestServer: @pytest.fixture def server_port(self): """Use a random available port for testing.""" - import socket - - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind(("127.0.0.1", 0)) - return s.getsockname()[1] + return get_free_port() @pytest.fixture def server(self, server_port): """Start the server in a background thread.""" - - def run_server(): - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - try: - loop.run_until_complete(start_tcp_server("127.0.0.1", server_port)) - except asyncio.CancelledError: - pass - finally: - loop.close() - - thread = threading.Thread(target=run_server, daemon=True) - thread.start() - time.sleep(0.5) + start_cli_server_thread(server_port) yield server_port - @pytest.fixture - def simple_script(self) -> str: - return """ -from dataclasses import dataclass - -@dataclass -class Input: - message: str - repeat: int = 1 - -def main(input: Input) -> str: - return (input.message + " ") * input.repeat -""" - def test_start_job_success(self, server, temp_dir, simple_script): """Test starting a job through the server.""" port = server @@ -140,6 +147,27 @@ def test_start_job_unknown_command(self, server): assert response["success"] is False assert "Unknown command" in response["error"] + def test_bad_working_directory_returns_400(self, server): + """A request-shaped error (bad working dir) is a 4xx, not a 200.""" + port = server + + async def send(): + async with aiohttp.ClientSession() as session: + async with session.post( + f"http://127.0.0.1:{port}/jobs/job-bad-cwd/start", + json={ + "command": "run", + "args": [], + "workingDirectory": "/no/such/dir/xyz-does-not-exist", + }, + ) as response: + return response.status, await response.json() + + status, body = asyncio.run(send()) + assert status == 400 + assert body["success"] is False + assert "working directory" in body["error"] + def test_start_job_missing_command(self, server): """Test starting a job without command field.""" port = server @@ -228,16 +256,98 @@ async def send_with_host(host: str): assert body == "OK" +class TestServerProcess: + """Tests for the real `uipath server` command startup path.""" + + @pytest.mark.skipif( + sys.platform == "win32", + reason="Windows GitHub runners fail before server startup while loading asyncio.", + ) + def test_uipath_server_process_starts_and_runs_job( + self, temp_dir: str, simple_script: str + ): + """The CLI command must preload modules, start the server, and run jobs.""" + port = get_free_port() + script_file = "entrypoint.py" + script_path = os.path.join(temp_dir, script_file) + input_file = os.path.join(temp_dir, "input.json") + output_file = os.path.join(temp_dir, "output.json") + + with open(script_path, "w") as f: + f.write(simple_script) + + with open(os.path.join(temp_dir, "uipath.json"), "w") as f: + json.dump(create_uipath_json(script_file), f) + + with open(input_file, "w") as f: + json.dump({"message": "Hello", "repeat": 2}, f) + + src_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "..", "src") + ) + env = os.environ.copy() + env["PYTHONPATH"] = os.pathsep.join( + part for part in [src_path, env.get("PYTHONPATH")] if part + ) + + process = subprocess.Popen( + [ + sys.executable, + "-c", + "from uipath._cli import cli; cli()", + "server", + "--tcp", + "--port", + str(port), + ], + cwd=temp_dir, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + + try: + wait_for_server_health(port, process) + + response = asyncio.run( + asyncio.wait_for( + start_job( + port, + "process-job-123", + "run", + [ + "main", + "--input-file", + input_file, + "--output-file", + output_file, + ], + ), + timeout=JOB_START_TIMEOUT_SECONDS, + ) + ) + + assert response["success"] is True + assert response["job_key"] == "process-job-123" + assert process.poll() is None + assert os.path.exists(output_file) + + with open(output_file, "r") as f: + output = f.read() + assert "Hello" in output + finally: + output = stop_process(process) + + assert "Traceback" not in output + + class TestServerEnvIsolation: """Test that environment variables are isolated between sequential server requests.""" @pytest.fixture def server_port(self): - import socket - - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind(("127.0.0.1", 0)) - return s.getsockname()[1] + return get_free_port() @pytest.fixture def env_snapshots(self): @@ -248,33 +358,21 @@ def server_with_spy(self, server_port, env_snapshots): """Start server with a spy command that captures os.environ.""" import click - from uipath._cli import cli_server + from uipath._cli import _server_core @click.command() def spy_cmd(): env_snapshots.append(dict(os.environ)) - original_commands = cli_server.COMMANDS.copy() - cli_server.COMMANDS["spy"] = spy_cmd + original_commands = _server_core.COMMANDS.copy() + _server_core.COMMANDS["spy"] = spy_cmd - def run_server(): - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - try: - loop.run_until_complete(start_tcp_server("127.0.0.1", server_port)) - except asyncio.CancelledError: - pass - finally: - loop.close() - - thread = threading.Thread(target=run_server, daemon=True) - thread.start() - time.sleep(0.5) + start_cli_server_thread(server_port) yield server_port - cli_server.COMMANDS.clear() - cli_server.COMMANDS.update(original_commands) + _server_core.COMMANDS.clear() + _server_core.COMMANDS.update(original_commands) def test_env_vars_do_not_leak_between_requests( self, server_with_spy, env_snapshots @@ -319,7 +417,7 @@ def test_env_vars_do_not_leak_between_requests( def test_server_baseline_env_preserved(self, server_with_spy, env_snapshots): """Server baseline env vars (like PATH) should be available during command execution.""" - from uipath._cli import cli_server + from uipath._cli import _server_core port = server_with_spy @@ -337,7 +435,7 @@ def test_server_baseline_env_preserved(self, server_with_spy, env_snapshots): env_run = env_snapshots[0] # Baseline is captured at server start, not import time - baseline = cli_server._state.baseline_env + baseline = _server_core._state.baseline_env assert baseline is not None # Baseline env vars should be present @@ -348,7 +446,7 @@ def test_server_baseline_env_preserved(self, server_with_spy, env_snapshots): def test_env_restored_after_request(self, server_with_spy): """os.environ should be restored to baseline after each request.""" - from uipath._cli import cli_server + from uipath._cli import _server_core port = server_with_spy @@ -362,10 +460,66 @@ def test_env_restored_after_request(self, server_with_spy): ) ) - baseline = cli_server._state.baseline_env + baseline = _server_core._state.baseline_env assert baseline is not None # After the request, os.environ should match baseline assert "SHOULD_NOT_PERSIST" not in os.environ for key in baseline: assert os.environ.get(key) == baseline[key] + + +class TestPreloadModules: + """Tests for preload_modules and its find_spec guard.""" + + def _run_with_modules(self, monkeypatch, modules): + from uipath._cli import cli_server + + class _FakeEntryPoint: + name = "fake" + + def load(self): + return lambda: modules + + monkeypatch.setattr( + cli_server, "entry_points", lambda group: [_FakeEntryPoint()] + ) + monkeypatch.setattr(cli_server, "DEFAULT_PRELOAD_MODULES", []) + cli_server.preload_modules() + + def test_missing_parent_package_does_not_crash(self, monkeypatch): + # find_spec raises ModuleNotFoundError when a parent package is absent; + # a stale entry like this must be skipped, not take down the server. + self._run_with_modules(monkeypatch, ["definitely_missing_pkg._private.types"]) + + def test_missing_leaf_module_is_skipped(self, monkeypatch): + # parent imports fine, leaf is absent -> find_spec returns None + self._run_with_modules(monkeypatch, ["json.does_not_exist"]) + + def test_existing_module_is_imported(self, monkeypatch): + import sys + + sys.modules.pop("difflib", None) + self._run_with_modules(monkeypatch, ["difflib"]) + assert "difflib" in sys.modules + + def test_already_loaded_module_is_skipped(self, monkeypatch): + import sys + + assert "json" in sys.modules + self._run_with_modules(monkeypatch, ["json"]) + + def test_failing_entry_point_does_not_crash(self, monkeypatch): + from uipath._cli import cli_server + + class _BrokenEntryPoint: + name = "broken" + + def load(self): + raise RuntimeError("boom") + + monkeypatch.setattr( + cli_server, "entry_points", lambda group: [_BrokenEntryPoint()] + ) + monkeypatch.setattr(cli_server, "DEFAULT_PRELOAD_MODULES", []) + cli_server.preload_modules() diff --git a/packages/uipath/tests/cli/test_server_ipc.py b/packages/uipath/tests/cli/test_server_ipc.py new file mode 100644 index 000000000..43fe313c9 --- /dev/null +++ b/packages/uipath/tests/cli/test_server_ipc.py @@ -0,0 +1,335 @@ +"""Tests for the uipath-ipc runtime server channel. + +The server hosts ``IPythonRuntimeServer`` (RunJob / StopJob) on a named pipe +alongside the HTTP channel when ``--ipc-pipe`` names one (see +``test_server_transport.py`` for the channel composition). Mirrors +``test_server.py`` (the HTTP path) but drives the pipe with a Python +``uipath-ipc`` client. + +Requires ``uipath-ipc`` to be installed. ``RunJob`` success runs the real +runtime (like ``test_server.test_start_job_success``); the rest exercise the IPC +wiring and env isolation without it. +""" + +import asyncio +import json +import os +import sys +import threading +import time +from typing import Any, Awaitable, Callable + +import click +import pytest +from uipath_ipc import ( + IpcClient, + IpcServer, + NamedPipeClientTransport, + NamedPipeServerTransport, +) + +from uipath._cli import _server_core +from uipath._cli.cli_server import ( + IPythonRuntimeServer, + RunJobResult, + start_ipc_server, +) + +_pipe_counter = 0 + + +def _unique_pipe() -> str: + global _pipe_counter + _pipe_counter += 1 + return f"uipath-ipc-test-{os.getpid()}-{_pipe_counter}" + + +def _serve_in_background(pipe_name: str) -> None: + """Run the IPC server on its own event loop in a daemon thread. + + ``asyncio.new_event_loop()`` yields the per-OS default loop — Proactor on + Windows (required for named pipes), Selector on Linux (CoreFxPipe UDS). + """ + + def run_server() -> None: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + loop.run_until_complete(start_ipc_server(pipe_name)) + except asyncio.CancelledError: + pass + finally: + loop.close() + + thread = threading.Thread(target=run_server, daemon=True) + thread.start() + _wait_until_ready(pipe_name) + + +async def _with_proxy(pipe_name: str, fn: Callable[[Any], Awaitable[Any]]) -> Any: + """Connect a uipath-ipc client to the pipe, run ``fn(proxy)``, then close.""" + client = IpcClient(transport=NamedPipeClientTransport(pipe_name)) + try: + # get_proxy is by-contract; the abstract interface is exactly what it wants. + proxy = client.get_proxy(IPythonRuntimeServer) # type: ignore[type-abstract] + return await fn(proxy) + finally: + await client.aclose() + + +def _wait_until_ready(pipe_name: str, timeout: float = 10.0) -> None: + """Poll the pipe until the IPC server answers, instead of a fixed sleep. + + A fixed ``sleep`` races the server's startup under load; this connects a real + client and calls the no-op ``StopJob`` until it succeeds (or times out). + """ + deadline = time.monotonic() + timeout + last_err: Exception | None = None + while time.monotonic() < deadline: + try: + asyncio.run( + _with_proxy( + pipe_name, + lambda p: p.StopJob( + {"JobKey": "00000000-0000-0000-0000-000000000000"} + ), + ) + ) + return + except Exception as e: # server not accepting connections yet + last_err = e + time.sleep(0.05) + raise TimeoutError( + f"IPC server on pipe {pipe_name!r} not ready within {timeout}s: {last_err}" + ) + + +def create_uipath_json( + script_path: str, entrypoint_name: str = "main" +) -> dict[str, Any]: + return {"functions": {entrypoint_name: f"{script_path}:main"}} + + +SIMPLE_SCRIPT = """ +from dataclasses import dataclass + +@dataclass +class Input: + message: str + repeat: int = 1 + +def main(input: Input) -> str: + return (input.message + " ") * input.repeat +""" + + +def test_start_ipc_server_fails_fast_without_uipath_ipc(monkeypatch): + """--ipc-pipe with uipath-ipc absent must fail loudly, not silently no-op.""" + monkeypatch.setitem(sys.modules, "uipath_ipc", None) + coro = start_ipc_server(_unique_pipe()) + with pytest.raises(RuntimeError, match="uipath-ipc"): + asyncio.run(coro) + + +class TestIpcServer: + @pytest.fixture + def pipe(self): + pipe_name = _unique_pipe() + _serve_in_background(pipe_name) + # Daemon thread; the server blocks in serve_forever and is torn down when + # the process exits (mirrors test_server.py's background HTTP server). + yield pipe_name + + def test_run_job_success(self, pipe, temp_dir): + """A real 'run' job executes and writes output.json (needs the runtime).""" + script_file = "entrypoint.py" + with open(os.path.join(temp_dir, script_file), "w") as f: + f.write(SIMPLE_SCRIPT) + with open(os.path.join(temp_dir, "uipath.json"), "w") as f: + json.dump(create_uipath_json(script_file), f) + + input_file = os.path.join(temp_dir, "input.json") + with open(input_file, "w") as f: + json.dump({"message": "Hello", "repeat": 3}, f) + output_file = os.path.join(temp_dir, "output.json") + + request = { + "JobKey": "job-123", + "Command": "run", + "Args": ["main", "--input-file", input_file, "--output-file", output_file], + "WorkingDirectory": temp_dir, + "EnvironmentVariables": {}, + } + result = asyncio.run(_with_proxy(pipe, lambda p: p.RunJob(request))) + + assert result.ExitCode == 0 + assert result.Error is None + assert os.path.exists(output_file) + with open(output_file, "r") as f: + assert "Hello" in f.read() + + def test_run_job_unknown_command(self, pipe): + request = {"JobKey": "job-1", "Command": "does_not_exist"} + result = asyncio.run(_with_proxy(pipe, lambda p: p.RunJob(request))) + assert result.ExitCode != 0 + assert "Unknown command" in (result.Error or "") + + def test_run_job_missing_command(self, pipe): + """Absent/empty Command is rejected before the job core is touched.""" + result = asyncio.run(_with_proxy(pipe, lambda p: p.RunJob({"JobKey": "job-1"}))) + assert result.ExitCode != 0 + assert "Command" in (result.Error or "") + + def test_run_job_accepts_resume_version(self, pipe): + request = { + "JobKey": "3f2504e0-4f89-11d3-9a0c-0305e82c3301", + "ResumeVersion": 4, + "Command": "does_not_exist", + } + result = asyncio.run(_with_proxy(pipe, lambda p: p.RunJob(request))) + + assert "Unknown command" in (result.Error or "") + + def test_stop_job_accepts_resume_version_and_force_stop(self, pipe): + request = { + "JobKey": "3f2504e0-4f89-11d3-9a0c-0305e82c3301", + "ResumeVersion": 2, + "ForceStop": True, + } + result = asyncio.run(_with_proxy(pipe, lambda p: p.StopJob(request))) + + assert result is True + + def test_stop_job_returns_true(self, pipe): + """StopJob is a no-op stub today, but must ack (bool) so the call is awaitable.""" + result = asyncio.run( + _with_proxy( + pipe, lambda p: p.StopJob({"JobKey": "job-1", "ForceStop": True}) + ) + ) + assert result is True + + +class TestIpcServerEnvIsolation: + """Env vars must not leak between sequential jobs (as on the HTTP path).""" + + @pytest.fixture + def pipe_with_spy(self): + env_snapshots: list[dict[str, str]] = [] + + @click.command() + def spy_cmd() -> None: + env_snapshots.append(dict(os.environ)) + + original = _server_core.COMMANDS.copy() + _server_core.COMMANDS["spy"] = spy_cmd + + pipe_name = _unique_pipe() + _serve_in_background(pipe_name) + try: + yield pipe_name, env_snapshots + finally: + _server_core.COMMANDS.clear() + _server_core.COMMANDS.update(original) + + def test_env_vars_do_not_leak_between_jobs(self, pipe_with_spy): + pipe_name, env_snapshots = pipe_with_spy + + async def run_two(proxy: Any) -> None: + await proxy.RunJob( + { + "JobKey": "job-1", + "Command": "spy", + "EnvironmentVariables": {"TEST_VAR_A": "a"}, + } + ) + await proxy.RunJob( + { + "JobKey": "job-2", + "Command": "spy", + "EnvironmentVariables": {"TEST_VAR_B": "b"}, + } + ) + + asyncio.run(_with_proxy(pipe_name, run_two)) + + assert len(env_snapshots) == 2 + run1, run2 = env_snapshots + assert run1["TEST_VAR_A"] == "a" + assert "TEST_VAR_B" not in run1 + assert run2["TEST_VAR_B"] == "b" + assert "TEST_VAR_A" not in run2 + + +class TestIpcContractFieldTransit: + @staticmethod + def _serve_spy(pipe_name: str, service: IPythonRuntimeServer) -> None: + def run_server() -> None: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + async def main() -> None: + server = IpcServer( + transport=NamedPipeServerTransport(pipe_name), + services={IPythonRuntimeServer: service}, + request_timeout=None, + ) + async with server: + await server.serve_forever() + + try: + loop.run_until_complete(main()) + except asyncio.CancelledError: + pass + finally: + loop.close() + + threading.Thread(target=run_server, daemon=True).start() + _wait_until_ready(pipe_name) + + def test_all_wire_fields_arrive_intact(self): + received: list[Any] = [] + + class SpyService(IPythonRuntimeServer): + async def RunJob(self, request: Any) -> RunJobResult: + received.append(request) + return RunJobResult(ExitCode=0) + + async def StopJob(self, request: Any) -> bool: + received.append(request) + return True + + pipe = _unique_pipe() + self._serve_spy(pipe, SpyService()) + received.clear() # drop the readiness probe's StopJob + + job_key = "3f2504e0-4f89-11d3-9a0c-0305e82c3301" + + async def drive(proxy: Any) -> None: + await proxy.RunJob( + { + "JobKey": job_key, + "ResumeVersion": 5, + "Command": "run", + "Args": "main --input-file in.json", + "WorkingDirectory": "/tmp/wd", + "EnvironmentVariables": {"A": "1"}, + } + ) + await proxy.StopJob( + {"JobKey": job_key, "ResumeVersion": 5, "ForceStop": True} + ) + + asyncio.run(_with_proxy(pipe, drive)) + + run_request, stop_request = received + assert run_request.JobKey == job_key + assert run_request.ResumeVersion == 5 + assert run_request.Command == "run" + assert run_request.Args == "main --input-file in.json" + assert run_request.WorkingDirectory == "/tmp/wd" + assert run_request.EnvironmentVariables == {"A": "1"} + + assert stop_request.JobKey == job_key + assert stop_request.ResumeVersion == 5 + assert stop_request.ForceStop is True diff --git a/packages/uipath/tests/cli/test_server_job_core.py b/packages/uipath/tests/cli/test_server_job_core.py new file mode 100644 index 000000000..1722d0c5b --- /dev/null +++ b/packages/uipath/tests/cli/test_server_job_core.py @@ -0,0 +1,94 @@ +"""Unit tests for ``_run_command_isolated`` — the shared job core behind both the +HTTP and uipath-ipc channels. These drive its error branches directly (bad +working dir, SystemExit, unexpected exception, uninitialized state) without +standing up a transport, so they run fast on every OS/Python. +""" + +import asyncio +import os +from typing import Any +from unittest.mock import Mock + +import pytest + +from uipath._cli import _server_core + + +@pytest.fixture +def restore_state(): + """Save/restore the module-level _ServerState singleton around a test.""" + saved_lock = _server_core._state.lock + saved_env = _server_core._state.baseline_env + try: + yield _server_core._state + finally: + _server_core._state.lock = saved_lock + _server_core._state.baseline_env = saved_env + + +def _init(state: Any) -> None: + state.lock = asyncio.Lock() + state.baseline_env = dict(os.environ) + + +async def test_requires_initialized_state(restore_state: Any) -> None: + restore_state.lock = None + restore_state.baseline_env = None + cmd = Mock() + with pytest.raises(RuntimeError, match="not initialized"): + await _server_core._run_command_isolated(cmd, [], {}, None) + + +async def test_rejects_bad_working_dir(restore_state: Any, tmp_path: Any) -> None: + _init(restore_state) + missing = str(tmp_path / "does-not-exist") + result = await _server_core._run_command_isolated(Mock(), [], {}, missing) + assert result["ExitCode"] == 1 + assert "working directory" in result["Error"] + assert result["Unexpected"] is False + assert result["ClientError"] is True # HTTP maps this to 400 + + +async def test_maps_system_exit_code(restore_state: Any) -> None: + _init(restore_state) + cmd = Mock() + cmd.main.side_effect = SystemExit(2) + result = await _server_core._run_command_isolated(cmd, [], {}, None) + assert result["ExitCode"] == 2 + assert result["Unexpected"] is False + + +async def test_reports_unexpected_exception(restore_state: Any) -> None: + _init(restore_state) + cmd = Mock() + cmd.main.side_effect = ValueError("boom") + result = await _server_core._run_command_isolated(cmd, [], {}, None) + assert result["ExitCode"] == 1 + assert result["Unexpected"] is True + assert "boom" in result["Error"] + + +# parse_args accepts what every caller sends: the .NET peer sends a single +# string (shlex-split), HTTP dicts / tests may send a pre-split list, or None. + + +def test_parse_args_splits_a_string() -> None: + # The real .NET-shaped Args: a single string, shlex-split into argv. + assert _server_core.parse_args("run --input-file in.json --flag") == [ + "run", + "--input-file", + "in.json", + "--flag", + ] + + +def test_parse_args_passes_a_list_through() -> None: + assert _server_core.parse_args(["run", "--input-file", "in.json"]) == [ + "run", + "--input-file", + "in.json", + ] + + +def test_parse_args_none_is_empty() -> None: + assert _server_core.parse_args(None) == [] diff --git a/packages/uipath/tests/cli/test_server_transport.py b/packages/uipath/tests/cli/test_server_transport.py new file mode 100644 index 000000000..6d6e2e489 --- /dev/null +++ b/packages/uipath/tests/cli/test_server_transport.py @@ -0,0 +1,216 @@ +"""`uipath server` serves BOTH transports concurrently — never either/or. + +The HTTP channel (aiohttp over a Unix socket, or TCP on Windows / ``--tcp``) is +ALWAYS started. The uipath-ipc named-pipe channel is opt-in and independent of the +HTTP socket: it is started alongside HTTP only when ``--ipc-pipe`` names a pipe, +served verbatim on that name (both sides agree on it out of band). The HTTP channel +is never torn down. + +These tests stub the three channel runners (so ``_serve``'s ``asyncio.gather`` +returns at once instead of serving forever) and assert which channels ``_serve`` +composes, how ``_run_server`` resolves its arguments, and that the CLI wires them +through. +""" + +import asyncio +from typing import Any + +from click.testing import CliRunner + +import uipath._cli._telemetry as _telemetry +from uipath._cli import _server_core, cli_server + + +def _stub_channels(monkeypatch) -> dict[str, Any]: + """Stub the three channel runners + state init; record what ``_serve`` starts. + + Each runner is replaced with an async recorder that returns immediately, so + ``_serve``'s ``asyncio.gather`` completes instead of blocking in serve-forever. + ``_state.init`` is stubbed too, so no event-loop-bound lock leaks between the + per-test loops ``asyncio.run`` creates. + """ + calls: dict[str, Any] = {} + + async def _rec_unix(ack_socket_path, server_socket_path=None): + calls["unix"] = (ack_socket_path, server_socket_path) + + async def _rec_tcp(host, port): + calls["tcp"] = (host, port) + + async def _rec_ipc(pipe_name): + calls["ipc"] = pipe_name + + monkeypatch.setattr(_server_core._state, "init", lambda: None) + monkeypatch.setattr(cli_server, "start_unix_server", _rec_unix) + monkeypatch.setattr(cli_server, "start_tcp_server", _rec_tcp) + monkeypatch.setattr(cli_server, "start_ipc_server", _rec_ipc) + return calls + + +# --------------------------------------------------------------------------- # +# _serve: channel composition # +# --------------------------------------------------------------------------- # + + +def test_serve_runs_http_and_ipc_together(monkeypatch): + calls = _stub_channels(monkeypatch) + asyncio.run( + cli_server._serve("/tmp/ack.sock", "/tmp/run-1.sock", "agent.pipe", 8765, False) + ) + assert calls["unix"] == ("/tmp/ack.sock", "/tmp/run-1.sock") + assert calls["ipc"] == "agent.pipe" # served on the explicit --ipc-pipe name + assert "tcp" not in calls + + +def test_serve_uses_the_explicit_ipc_pipe_name(monkeypatch): + """The IPC pipe name is taken verbatim from ``--ipc-pipe``, not derived from the + HTTP socket path.""" + calls = _stub_channels(monkeypatch) + asyncio.run( + cli_server._serve( + "/tmp/ack.sock", + "/var/tmp/uipath-server-42.sock", + "my-agent-pipe", + 8765, + False, + ) + ) + assert calls["ipc"] == "my-agent-pipe" # verbatim, independent of the HTTP socket + + +def test_serve_ipc_is_independent_of_the_http_socket(monkeypatch): + """``--ipc-pipe`` drives IPC even when HTTP auto-generates its socket + (``server_socket`` is ``None``).""" + calls = _stub_channels(monkeypatch) + asyncio.run(cli_server._serve("/tmp/ack.sock", None, "agent.pipe", 8765, False)) + assert calls["unix"] == ("/tmp/ack.sock", None) # HTTP auto-socket + assert calls["ipc"] == "agent.pipe" + + +def test_serve_rides_ipc_alongside_tcp(monkeypatch): + calls = _stub_channels(monkeypatch) + asyncio.run( + cli_server._serve("/tmp/ack.sock", "/tmp/run-1.sock", "agent.pipe", 9000, True) + ) + assert calls["tcp"] == ("127.0.0.1", 9000) + assert "unix" not in calls + assert calls["ipc"] == "agent.pipe" # IPC rides next to TCP too, not only UDS + + +def test_serve_skips_ipc_without_ipc_pipe(monkeypatch): + """No ``--ipc-pipe`` ⇒ HTTP only, regardless of the HTTP socket.""" + calls = _stub_channels(monkeypatch) + asyncio.run( + cli_server._serve("/tmp/ack.sock", "/tmp/run-1.sock", None, 8765, False) + ) + assert calls["unix"] == ("/tmp/ack.sock", "/tmp/run-1.sock") + assert "ipc" not in calls + + +# --------------------------------------------------------------------------- # +# _run_server: argument resolution + per-OS loop # +# --------------------------------------------------------------------------- # + + +def _capture_serve(monkeypatch) -> dict[str, Any]: + """Replace ``_serve`` with an async recorder so ``_run_server`` runs to + completion on its real per-OS loop (Proactor on Windows, ``asyncio.run`` on + Linux) without actually serving anything.""" + seen: dict[str, Any] = {} + + async def _rec_serve(ack_socket_path, server_socket, ipc_pipe, port, use_tcp): + seen.update( + ack=ack_socket_path, + server_socket=server_socket, + ipc_pipe=ipc_pipe, + port=port, + use_tcp=use_tcp, + ) + + monkeypatch.setattr(cli_server, "_serve", _rec_serve) + return seen + + +def test_run_server_defaults_ack_from_env(monkeypatch): + seen = _capture_serve(monkeypatch) + monkeypatch.setenv(cli_server.SOCKET_ENV_VAR, "/tmp/from-env.sock") + cli_server._run_server(None, "/tmp/s.sock", "agent.pipe", None, False) + assert seen["ack"] == "/tmp/from-env.sock" + assert seen["server_socket"] == "/tmp/s.sock" + assert seen["ipc_pipe"] == "agent.pipe" + assert seen["port"] == cli_server.DEFAULT_PORT + assert seen["use_tcp"] is cli_server.IS_WINDOWS # UDS on Linux, TCP on Windows + + +def test_run_server_prefers_explicit_client_socket(monkeypatch): + seen = _capture_serve(monkeypatch) + monkeypatch.setenv(cli_server.SOCKET_ENV_VAR, "/tmp/from-env.sock") + cli_server._run_server("/tmp/explicit.sock", "/tmp/s.sock", None, 1234, False) + assert seen["ack"] == "/tmp/explicit.sock" # explicit arg beats the env var + assert seen["port"] == 1234 + + +def test_run_server_falls_back_to_default_ack(monkeypatch): + seen = _capture_serve(monkeypatch) + monkeypatch.delenv(cli_server.SOCKET_ENV_VAR, raising=False) + cli_server._run_server(None, "/tmp/s.sock", None, None, False) + assert seen["ack"] == cli_server.DEFAULT_SOCKET_PATH + + +def test_run_server_tcp_flag_forces_tcp(monkeypatch): + seen = _capture_serve(monkeypatch) + cli_server._run_server("/tmp/a.sock", "/tmp/s.sock", None, None, True) + assert seen["use_tcp"] is True + + +# --------------------------------------------------------------------------- # +# CLI wiring # +# --------------------------------------------------------------------------- # + + +def _stub_cli(monkeypatch) -> dict[str, Any]: + """Disable telemetry + preload and record the args the CLI hands _run_server.""" + seen: dict[str, Any] = {} + monkeypatch.setattr(_telemetry, "is_telemetry_enabled", lambda: False) + monkeypatch.setattr(cli_server, "preload_modules", lambda: None) + monkeypatch.setattr( + cli_server, + "_run_server", + lambda client_socket, server_socket, ipc_pipe, port, tcp: seen.update( + client_socket=client_socket, + server_socket=server_socket, + ipc_pipe=ipc_pipe, + port=port, + tcp=tcp, + ), + ) + return seen + + +def test_cli_passes_socket_args_through(monkeypatch): + seen = _stub_cli(monkeypatch) + result = CliRunner().invoke( + cli_server.server, + [ + "--client-socket", + "/tmp/ack.sock", + "--server-socket", + "/tmp/run.sock", + "--ipc-pipe", + "agent.pipe", + ], + ) + assert result.exit_code == 0, result.output + assert seen["client_socket"] == "/tmp/ack.sock" + assert seen["server_socket"] == "/tmp/run.sock" + assert seen["ipc_pipe"] == "agent.pipe" + assert seen["tcp"] is False + + +def test_cli_ipc_pipe_is_optional(monkeypatch): + """No ``--ipc-pipe`` ⇒ HTTP only; the server still starts (IPC simply skipped).""" + seen = _stub_cli(monkeypatch) + result = CliRunner().invoke(cli_server.server, []) + assert result.exit_code == 0, result.output + assert seen["server_socket"] is None + assert seen["ipc_pipe"] is None diff --git a/packages/uipath/tests/cli/unit/test_runtime_protocol_compat.py b/packages/uipath/tests/cli/unit/test_runtime_protocol_compat.py new file mode 100644 index 000000000..812ddaa11 --- /dev/null +++ b/packages/uipath/tests/cli/unit/test_runtime_protocol_compat.py @@ -0,0 +1,139 @@ +from collections.abc import AsyncGenerator +from datetime import datetime, timezone +from typing import Any + +import pytest + +from uipath.core.triggers import ( + UiPathResumeTrigger, + UiPathResumeTriggerName, + UiPathResumeTriggerType, +) +from uipath.platform.common import WaitUntil +from uipath.platform.resume_triggers import UiPathResumeTriggerHandler +from uipath.runtime import ( + UiPathExecuteOptions, + UiPathResumableRuntime, + UiPathResumableStorageProtocol, + UiPathResumeTriggerProtocol, + UiPathRuntimeEvent, + UiPathRuntimeProtocol, + UiPathRuntimeResult, + UiPathRuntimeStatus, + UiPathStreamOptions, +) +from uipath.runtime.schema import UiPathRuntimeSchema + + +class SuspendedRuntime: + async def execute( + self, + input: dict[str, Any] | None = None, + options: UiPathExecuteOptions | None = None, + ) -> UiPathRuntimeResult: + del input, options + return UiPathRuntimeResult( + status=UiPathRuntimeStatus.SUSPENDED, + output={ + "interrupt-1": WaitUntil( + resume_time=datetime(2026, 7, 6, 12, tzinfo=timezone.utc) + ) + }, + ) + + async def stream( + self, + input: dict[str, Any] | None = None, + options: UiPathStreamOptions | None = None, + ) -> AsyncGenerator[UiPathRuntimeEvent, None]: + yield await self.execute(input, options) + + async def get_schema(self) -> UiPathRuntimeSchema: + return UiPathRuntimeSchema( + filePath="agent.py", + uniqueId="agent", + type="test", + input={}, + output={}, + ) + + async def dispose(self) -> None: + pass + + +class MemoryResumableStorage: + def __init__(self) -> None: + self.triggers: list[UiPathResumeTrigger] | None = None + self.saved_runtime_id: str | None = None + + async def set_value( + self, runtime_id: str, namespace: str, key: str, value: Any + ) -> None: + del runtime_id, namespace, key, value + + async def get_value(self, runtime_id: str, namespace: str, key: str) -> Any: + del runtime_id, namespace, key + return None + + async def save_triggers( + self, runtime_id: str, triggers: list[UiPathResumeTrigger] + ) -> None: + self.saved_runtime_id = runtime_id + self.triggers = triggers + + async def get_triggers(self, runtime_id: str) -> list[UiPathResumeTrigger] | None: + del runtime_id + return self.triggers + + async def delete_triggers( + self, runtime_id: str, triggers: list[UiPathResumeTrigger] + ) -> None: + del runtime_id + if self.triggers is None: + return + self.triggers = [ + trigger for trigger in self.triggers if trigger not in triggers + ] + + async def delete_trigger( + self, runtime_id: str, trigger: UiPathResumeTrigger + ) -> None: + await self.delete_triggers(runtime_id, [trigger]) + + +@pytest.mark.anyio +async def test_platform_handler_satisfies_runtime_trigger_protocol() -> None: + handler: UiPathResumeTriggerProtocol = UiPathResumeTriggerHandler() + + triggers = await handler.create_triggers( + WaitUntil(resume_time=datetime(2026, 7, 6, 12, tzinfo=timezone.utc)) + ) + + assert len(triggers) == 1 + assert triggers[0].trigger_type == UiPathResumeTriggerType.TIMER + assert triggers[0].trigger_name == UiPathResumeTriggerName.TIMER + + +@pytest.mark.anyio +async def test_resumable_runtime_uses_platform_trigger_protocol() -> None: + delegate: UiPathRuntimeProtocol = SuspendedRuntime() + memory_storage = MemoryResumableStorage() + storage: UiPathResumableStorageProtocol = memory_storage + handler: UiPathResumeTriggerProtocol = UiPathResumeTriggerHandler() + + runtime = UiPathResumableRuntime( + delegate=delegate, + storage=storage, + trigger_manager=handler, + runtime_id="runtime-1", + ) + + result = await runtime.execute() + + assert result.status == UiPathRuntimeStatus.SUSPENDED + assert memory_storage.saved_runtime_id == "runtime-1" + assert result.triggers is not None + assert len(result.triggers) == 1 + assert result.triggers[0].interrupt_id == "interrupt-1" + assert result.triggers[0].trigger_type == UiPathResumeTriggerType.TIMER + assert result.triggers[0].trigger_name == UiPathResumeTriggerName.TIMER diff --git a/packages/uipath/tests/cli/unit/test_runtime_protocol_conformance.py b/packages/uipath/tests/cli/unit/test_runtime_protocol_conformance.py new file mode 100644 index 000000000..05697dbb4 --- /dev/null +++ b/packages/uipath/tests/cli/unit/test_runtime_protocol_conformance.py @@ -0,0 +1,79 @@ +"""Protocol conformance for uipath-runtime protocol implementations. + +Every protocol-annotated assignment below is a typed boundary: mypy verifies +the implementation against the protocol surface of the installed +uipath-runtime version. A dependency bump that adds or changes protocol +members fails typecheck on these lines until the implementations catch up. + +Deliberately construction-only: no runtime behavior is exercised here. +""" + +import uuid + +from uipath.core.events import EventBus +from uipath.core.tracing import UiPathTraceManager +from uipath.eval.runtime import UiPathEvalContext, UiPathEvalRuntime +from uipath.functions import ( + UiPathDebugFunctionsRuntime, + UiPathFunctionsRuntime, + UiPathFunctionsRuntimeFactory, +) +from uipath.platform.resume_triggers import ( + UiPathResumeTriggerCreator, + UiPathResumeTriggerReader, +) +from uipath.runtime import ( + UiPathRuntimeFactoryProtocol, + UiPathRuntimeProtocol, +) +from uipath.runtime.resumable.protocols import ( + UiPathResumeTriggerCreatorProtocol, + UiPathResumeTriggerReaderProtocol, +) + + +def test_functions_runtime_satisfies_runtime_protocol() -> None: + runtime: UiPathRuntimeProtocol = UiPathFunctionsRuntime( + file_path="main.py", + function_name="main", + entrypoint_name="main", + ) + + # the debug wrapper's `delegate` parameter is protocol-typed: passing the + # real functions runtime through it is a second typed boundary, and the + # wrapper itself must satisfy the protocol too + debug_runtime: UiPathRuntimeProtocol = UiPathDebugFunctionsRuntime( + delegate=runtime, + entrypoint_path="main.py", + function_name="main", + ) + + assert debug_runtime is not None + + +def test_functions_factory_satisfies_factory_protocol() -> None: + factory: UiPathRuntimeFactoryProtocol = UiPathFunctionsRuntimeFactory() + + # the eval runtime's `factory` parameter is protocol-typed: the real + # functions factory flows through it. UiPathEvalRuntime itself is a + # specialized runtime (no stream, argument-less execute) and is + # deliberately NOT pinned against UiPathRuntimeProtocol. + context = UiPathEvalContext() + context.execution_id = str(uuid.uuid4()) + eval_runtime = UiPathEvalRuntime( + context=context, + factory=factory, + trace_manager=UiPathTraceManager(), + event_bus=EventBus(), + ) + + assert eval_runtime is not None + + +def test_standalone_trigger_classes_satisfy_protocols() -> None: + # cover direct consumers of the standalone creator/reader classes + creator: UiPathResumeTriggerCreatorProtocol = UiPathResumeTriggerCreator() + reader: UiPathResumeTriggerReaderProtocol = UiPathResumeTriggerReader() + + assert creator is not None + assert reader is not None diff --git a/packages/uipath/tests/cli/utils/server.py b/packages/uipath/tests/cli/utils/server.py new file mode 100644 index 000000000..abf1681f8 --- /dev/null +++ b/packages/uipath/tests/cli/utils/server.py @@ -0,0 +1,93 @@ +import asyncio +import socket +import threading +import time +from typing import Any +from urllib.request import urlopen + +import aiohttp + +from uipath._cli.cli_server import start_tcp_server +from uipath.functions import register_default_runtime_factory + + +def get_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def _wait_for_health(port: int, timeout: float = 10.0) -> None: + deadline = time.monotonic() + timeout + last_error: Exception | None = None + while time.monotonic() < deadline: + try: + with urlopen(f"http://127.0.0.1:{port}/health", timeout=0.5) as response: + if response.status == 200: + return + except Exception as exc: + last_error = exc + time.sleep(0.05) + raise RuntimeError(f"server on port {port} did not become healthy") from last_error + + +def start_cli_server_thread(port: int) -> threading.Thread: + register_default_runtime_factory() + + def run_server() -> None: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + loop.run_until_complete(start_tcp_server("127.0.0.1", port)) + except asyncio.CancelledError: + pass + finally: + loop.close() + + thread = threading.Thread(target=run_server, daemon=True) + thread.start() + _wait_for_health(port) + return thread + + +async def start_job( + port: int, + job_key: str, + command: str, + args: list[str], + working_directory: str | None = None, +) -> dict[str, Any]: + payload: dict[str, Any] = {"command": command, "args": args} + if working_directory is not None: + payload["workingDirectory"] = working_directory + + async with aiohttp.ClientSession() as session: + async with session.post( + f"http://127.0.0.1:{port}/jobs/{job_key}/start", + json=payload, + ) as response: + return await response.json() + + +async def start_job_with_env( + port: int, + job_key: str, + command: str, + args: list[str], + env_vars: dict[str, str], + working_directory: str | None = None, +) -> dict[str, Any]: + payload: dict[str, Any] = { + "command": command, + "args": args, + "environmentVariables": env_vars, + } + if working_directory is not None: + payload["workingDirectory"] = working_directory + + async with aiohttp.ClientSession() as session: + async with session.post( + f"http://127.0.0.1:{port}/jobs/{job_key}/start", + json=payload, + ) as response: + return await response.json() diff --git a/packages/uipath/tests/evaluators/test_dataset_classification_evaluators.py b/packages/uipath/tests/evaluators/test_dataset_classification_evaluators.py new file mode 100644 index 000000000..c4d427fe4 --- /dev/null +++ b/packages/uipath/tests/evaluators/test_dataset_classification_evaluators.py @@ -0,0 +1,648 @@ +"""Tests for dataset-level classification evaluators (Precision, Recall, FScore). + +Covers the math (2-class, 3-class, micro vs macro, F-beta), edge cases +(empty input, out-of-vocab labels, malformed details), factory dispatch, and +runtime-level routing where compute_dataset_evaluator_results walks +per-datapoint evaluator configs' embedded ``aggregators`` lists. +""" + +import uuid + +import pytest +from pydantic import BaseModel + +from uipath.eval.evaluators._aggregator_specs import ( + ConfusionMatrixAggregatorSpec, + FScoreAggregatorSpec, + PrecisionAggregatorSpec, + RecallAggregatorSpec, +) +from uipath.eval.evaluators.base_evaluator import BaseEvaluatorJustification +from uipath.eval.evaluators.classification_dataset_evaluators import ( + AveragedMetrics, + ClassificationDatasetEvaluator, + ClassificationDetails, + PerClassMetrics, +) +from uipath.eval.evaluators.dataset_evaluator_factory import build_dataset_evaluator +from uipath.eval.evaluators.exact_match_evaluator import ExactMatchEvaluator +from uipath.eval.models.models import ( + EvaluationResultDto, + NumericEvaluationResult, +) +from uipath.eval.runtime._types import ( + UiPathEvalRunResult, + UiPathEvalRunResultDto, +) +from uipath.eval.runtime.runtime import compute_dataset_evaluator_results + + +def _result(expected: str, actual: str) -> EvaluationResultDto: + """Build an EvaluationResultDto carrying an expected/actual justification.""" + justification = BaseEvaluatorJustification(expected=expected, actual=actual) + return EvaluationResultDto( + score=1.0 if expected.lower() == actual.lower() else 0.0, + details=justification.model_dump(), + ) + + +def _precision( + classes: list[str], averaging: str = "macro" +) -> ClassificationDatasetEvaluator: + spec = PrecisionAggregatorSpec(averaging=averaging) # type: ignore[arg-type] + return ClassificationDatasetEvaluator( + spec, source_evaluator="intent_match", classes=classes + ) + + +def _recall( + classes: list[str], averaging: str = "macro" +) -> ClassificationDatasetEvaluator: + spec = RecallAggregatorSpec(averaging=averaging) # type: ignore[arg-type] + return ClassificationDatasetEvaluator( + spec, source_evaluator="intent_match", classes=classes + ) + + +def _fscore( + classes: list[str], averaging: str = "macro", f_value: float = 1.0 +) -> ClassificationDatasetEvaluator: + spec = FScoreAggregatorSpec( + averaging=averaging, # type: ignore[arg-type] + f_value=f_value, + ) + return ClassificationDatasetEvaluator( + spec, source_evaluator="intent_match", classes=classes + ) + + +def _details(result: object) -> ClassificationDetails: + """Type-narrowing helper for asserting on details.""" + assert isinstance(result, NumericEvaluationResult) + assert isinstance(result.details, ClassificationDetails) + return result.details + + +# per_class / macro / micro are Optional on ClassificationDetails (the +# confusion_matrix variant omits them). Scalar-metric tests always populate +# them; these accessors assert-narrow to the non-Optional type so mypy is happy. +def _pc(d: ClassificationDetails) -> dict[str, PerClassMetrics]: + assert d.per_class is not None + return d.per_class + + +def _macro(d: ClassificationDetails) -> AveragedMetrics: + assert d.macro is not None + return d.macro + + +def _micro(d: ClassificationDetails) -> AveragedMetrics: + assert d.micro is not None + return d.micro + + +def _exact_match_evaluator( + name: str, + classes: list[str], + aggregators: list[BaseModel], +) -> ExactMatchEvaluator: + """Build a per-datapoint ExactMatch evaluator with attached aggregators. + + Aggregators + classes live on the evaluator config — every aggregator on + the same ExactMatch config shares the ``classes`` vocabulary declared here. + """ + return ExactMatchEvaluator.model_validate( + { + "id": str(uuid.uuid4()), + "evaluatorConfig": { + "name": name, + "classes": classes, + "aggregators": [spec.model_dump(by_alias=True) for spec in aggregators], + }, + } + ) + + +class TestPrecisionEvaluator: + def test_empty_input_returns_zeroed_result(self) -> None: + result = _precision(["cat", "dog"]).evaluate([]) + assert isinstance(result, NumericEvaluationResult) + assert result.score == 0.0 + d = _details(result) + assert d.n_total == 0 and d.n_scored == 0 + assert d.confusion_matrix == [[0, 0], [0, 0]] + assert _pc(d)["cat"].tp == 0 + assert _pc(d)["cat"].tn == 0 + + def test_confusion_matrix_is_predicted_by_expected(self) -> None: + # Pin the documented orientation: confusion_matrix[predicted][expected]. + # Differs from sklearn's [true][predicted] convention. + results = [ + _result("cat", "cat"), # expected=cat, predicted=cat -> [cat][cat] + _result("cat", "dog"), # expected=cat, predicted=dog -> [dog][cat] + _result("dog", "dog"), # expected=dog, predicted=dog -> [dog][dog] + _result("dog", "dog"), + ] + d = _details(_precision(["cat", "dog"]).evaluate(results)) + # classes -> index: cat=0, dog=1 + # [predicted=cat][expected=cat] = 1 + assert d.confusion_matrix[0][0] == 1 + # [predicted=dog][expected=cat] = 1 (the FP for dog / FN for cat) + assert d.confusion_matrix[1][0] == 1 + # [predicted=dog][expected=dog] = 2 + assert d.confusion_matrix[1][1] == 2 + # [predicted=cat][expected=dog] = 0 + assert d.confusion_matrix[0][1] == 0 + + def test_precision_two_class_macro(self) -> None: + results = [ + _result("yes", "yes"), + _result("yes", "yes"), + _result("yes", "no"), + _result("no", "yes"), + ] + result = _precision(["yes", "no"], averaging="macro").evaluate(results) + d = _details(result) + # precision_yes = 2 / (2 + 1) = 2/3 + # precision_no = 0 / (0 + 1) = 0 + # macro = (2/3 + 0) / 2 = 1/3 + assert _pc(d)["yes"].precision == pytest.approx(2 / 3) + assert _pc(d)["no"].precision == pytest.approx(0.0) + assert _macro(d).precision == pytest.approx((2 / 3 + 0.0) / 2) + assert result.score == pytest.approx(_macro(d).precision) + + def test_two_class_micro_equals_accuracy(self) -> None: + results = [ + _result("yes", "yes"), + _result("yes", "yes"), + _result("yes", "no"), + _result("no", "yes"), + ] + result = _precision(["yes", "no"], averaging="micro").evaluate(results) + d = _details(result) + assert _micro(d).precision == pytest.approx(0.5) + assert result.score == pytest.approx(0.5) + + def test_three_class_macro(self) -> None: + pairs = [ + ("cat", "cat"), + ("cat", "cat"), + ("cat", "dog"), + ("dog", "dog"), + ("dog", "dog"), + ("dog", "bird"), + ("bird", "bird"), + ("bird", "bird"), + ("bird", "cat"), + ] + result = _precision(["cat", "dog", "bird"], averaging="macro").evaluate( + [_result(e, a) for e, a in pairs] + ) + d = _details(result) + for label in ("cat", "dog", "bird"): + m = _pc(d)[label] + assert m.tp == 2 and m.fp == 1 and m.fn == 1 and m.tn == 5 + assert m.precision == pytest.approx(2 / 3) + assert _macro(d).precision == pytest.approx(2 / 3) + assert result.score == pytest.approx(2 / 3) + + +class TestRecallEvaluator: + def test_recall_two_class_macro(self) -> None: + results = [ + _result("yes", "yes"), + _result("yes", "yes"), + _result("yes", "no"), + _result("no", "yes"), + ] + result = _recall(["yes", "no"], averaging="macro").evaluate(results) + d = _details(result) + assert _pc(d)["yes"].recall == pytest.approx(2 / 3) + assert _pc(d)["no"].recall == pytest.approx(0.0) + assert result.score == pytest.approx(1 / 3) + + def test_recall_differs_from_precision(self) -> None: + results = [ + _result("yes", "yes"), + _result("yes", "yes"), + _result("no", "yes"), + _result("no", "yes"), + _result("no", "no"), + ] + p = _details(_precision(["yes", "no"], averaging="macro").evaluate(results)) + r = _details(_recall(["yes", "no"], averaging="macro").evaluate(results)) + assert _pc(p)["yes"].precision == pytest.approx(0.5) + assert _pc(p)["no"].precision == pytest.approx(1.0) + assert _pc(r)["yes"].recall == pytest.approx(1.0) + assert _pc(r)["no"].recall == pytest.approx(1 / 3) + + +class TestFScoreEvaluator: + def test_f1_equals_harmonic_mean_of_p_and_r(self) -> None: + results = [ + _result("yes", "yes"), + _result("yes", "yes"), + _result("yes", "no"), + _result("no", "yes"), + ] + f = _details( + _fscore(["yes", "no"], averaging="macro", f_value=1.0).evaluate(results) + ) + assert _pc(f)["yes"].f_score == pytest.approx(2 / 3) + assert _pc(f)["no"].f_score == pytest.approx(0.0) + assert _macro(f).f_score == pytest.approx((2 / 3 + 0.0) / 2) + + def test_f_beta_emphasizes_recall_when_beta_above_one(self) -> None: + results = [ + _result("yes", "yes"), + _result("yes", "yes"), + _result("no", "yes"), + _result("no", "yes"), + _result("no", "no"), + ] + f1 = _details( + _fscore(["yes", "no"], averaging="macro", f_value=1.0).evaluate(results) + ) + f2 = _details( + _fscore(["yes", "no"], averaging="macro", f_value=2.0).evaluate(results) + ) + assert _pc(f2)["yes"].f_score > _pc(f1)["yes"].f_score + + def test_three_class_micro_pools_across_classes(self) -> None: + pairs = [ + ("cat", "cat"), + ("cat", "cat"), + ("cat", "dog"), + ("dog", "dog"), + ("dog", "dog"), + ("dog", "bird"), + ("bird", "bird"), + ("bird", "bird"), + ("bird", "cat"), + ] + d = _details( + _fscore(["cat", "dog", "bird"], averaging="micro", f_value=1.0).evaluate( + [_result(e, a) for e, a in pairs] + ) + ) + assert _micro(d).f_score == pytest.approx(6 / 9) + + +class TestSkippingAndEdgeCases: + def test_out_of_vocab_prediction_counts_as_recall_miss(self) -> None: + results = [ + _result("cat", "cat"), + _result("cat", "platypus"), + ] + d = _details(_recall(["cat", "dog"]).evaluate(results)) + assert d.n_total == 2 and d.n_scored == 2 and d.n_skipped == 0 + assert _pc(d)["cat"].fn == 1 + assert _pc(d)["cat"].support == 2 + assert _pc(d)["cat"].recall == pytest.approx(0.5) + assert _pc(d)["cat"].precision == pytest.approx(1.0) + + def test_out_of_vocab_expected_label_is_skipped(self) -> None: + results = [ + _result("cat", "cat"), + _result("zebra", "dog"), + ] + d = _details(_precision(["cat", "dog"]).evaluate(results)) + assert d.n_total == 2 and d.n_scored == 1 and d.n_skipped == 1 + + def test_results_without_justification_are_skipped(self) -> None: + results = [ + _result("cat", "cat"), + EvaluationResultDto(score=1.0, details="just a string"), + EvaluationResultDto(score=0.0, details={"unrelated": "shape"}), + ] + d = _details(_precision(["cat", "dog"]).evaluate(results)) + assert d.n_total == 3 and d.n_scored == 1 and d.n_skipped == 2 + + def test_case_insensitive(self) -> None: + results = [_result("Cat", "CAT"), _result("DOG", "dog")] + d = _details(_precision(["cat", "dog"]).evaluate(results)) + assert _pc(d)["cat"].tp == 1 + assert _pc(d)["dog"].tp == 1 + + +class TestFactory: + """The factory builds from an AggregatorSpec instance + source name.""" + + def test_builds_precision_from_spec(self) -> None: + spec = PrecisionAggregatorSpec(averaging="macro") + evaluator = build_dataset_evaluator(spec, "intent_match", classes=["yes", "no"]) + assert isinstance(evaluator, ClassificationDatasetEvaluator) + assert evaluator.spec.type == "precision" + assert evaluator.source_evaluator == "intent_match" + + def test_builds_recall_from_spec(self) -> None: + spec = RecallAggregatorSpec(averaging="micro") + evaluator = build_dataset_evaluator(spec, "intent_match", classes=["yes", "no"]) + assert isinstance(evaluator, ClassificationDatasetEvaluator) + assert evaluator.spec.type == "recall" + + def test_builds_fscore_from_spec(self) -> None: + spec = FScoreAggregatorSpec(averaging="macro", f_value=2.0) + evaluator = build_dataset_evaluator(spec, "intent_match", classes=["yes", "no"]) + assert isinstance(evaluator, ClassificationDatasetEvaluator) + assert isinstance(evaluator.spec, FScoreAggregatorSpec) + assert evaluator.spec.f_value == 2.0 + + +class TestAggregatorSpecJsonRoundTrip: + """Pin the wire shape sent to the C# side.""" + + def test_precision_spec_wire_shape(self) -> None: + """Specs carry only metric-shape fields; ``classes`` lives on the + parent evaluator config. + """ + spec = PrecisionAggregatorSpec.model_validate( + { + "type": "precision", + "averaging": "macro", + } + ) + dumped = spec.model_dump(by_alias=True) + assert dumped == { + "type": "precision", + "averaging": "macro", + } + + def test_fscore_uses_camelcase_fvalue_on_wire(self) -> None: + spec = FScoreAggregatorSpec.model_validate( + { + "type": "fscore", + "averaging": "macro", + "fValue": 1.5, + } + ) + assert spec.f_value == 1.5 + dumped = spec.model_dump(by_alias=True) + assert dumped["fValue"] == 1.5 + assert "f_value" not in dumped + + def test_exact_match_evaluator_round_trips_aggregators(self) -> None: + """Per-datapoint evaluator config carries aggregators[]; survives dump+load.""" + ev = _exact_match_evaluator( + "intent_classifier", + classes=["book", "cancel", "reschedule"], + aggregators=[ + PrecisionAggregatorSpec(averaging="macro"), + FScoreAggregatorSpec(averaging="macro", f_value=1.0), + ], + ) + assert ev.evaluator_config.aggregators is not None + assert len(ev.evaluator_config.aggregators) == 2 + assert ev.evaluator_config.aggregators[0].type == "precision" + assert ev.evaluator_config.aggregators[1].type == "fscore" + + +class TestComputeDatasetEvaluatorResults: + """End-to-end: runtime walks evaluator configs' aggregators[].""" + + def test_walks_aggregators_on_classification_evaluator(self) -> None: + evaluator = _exact_match_evaluator( + "intent_match", + classes=["yes", "no"], + aggregators=[ + PrecisionAggregatorSpec(averaging="macro"), + RecallAggregatorSpec(averaging="macro"), + ], + ) + + eval_results = [ + UiPathEvalRunResult( + evaluation_name="dp1", + evaluation_run_results=[ + UiPathEvalRunResultDto( + evaluator_name="intent_match", + evaluator_id=str(uuid.uuid4()), + result=_result("yes", "yes"), + ), + UiPathEvalRunResultDto( + evaluator_name="some_other_evaluator", + evaluator_id=str(uuid.uuid4()), + result=EvaluationResultDto(score=0.5), + ), + ], + ), + UiPathEvalRunResult( + evaluation_name="dp2", + evaluation_run_results=[ + UiPathEvalRunResultDto( + evaluator_name="intent_match", + evaluator_id=str(uuid.uuid4()), + result=_result("yes", "no"), + ), + ], + ), + ] + + out = compute_dataset_evaluator_results(eval_results, [evaluator]) + # Two aggregators on intent_match → two keys, prefixed by source name. + assert set(out) == {"intent_match::precision", "intent_match::recall"} + precision_dto = out["intent_match::precision"] + assert isinstance(precision_dto, EvaluationResultDto) + assert isinstance(precision_dto.details, dict) + # The unrelated 0.5 score from some_other_evaluator must NOT be in the matrix. + assert precision_dto.details["nScored"] == 2 + + def test_evaluator_without_aggregators_is_skipped(self) -> None: + evaluator = _exact_match_evaluator( + "intent_match", classes=["yes", "no"], aggregators=[] + ) + eval_results = [ + UiPathEvalRunResult( + evaluation_name="dp1", + evaluation_run_results=[ + UiPathEvalRunResultDto( + evaluator_name="intent_match", + evaluator_id=str(uuid.uuid4()), + result=_result("yes", "yes"), + ), + ], + ), + ] + out = compute_dataset_evaluator_results(eval_results, [evaluator]) + assert out == {} + + def test_line_by_line_subresults_are_excluded(self) -> None: + evaluator = _exact_match_evaluator( + "intent_match", + classes=["yes", "no"], + aggregators=[ + PrecisionAggregatorSpec(averaging="macro"), + ], + ) + eval_results = [ + UiPathEvalRunResult( + evaluation_name="dp1", + evaluation_run_results=[ + UiPathEvalRunResultDto( + evaluator_name="intent_match", + evaluator_id=str(uuid.uuid4()), + result=_result("yes", "yes"), + is_line_result=True, + ), + UiPathEvalRunResultDto( + evaluator_name="intent_match", + evaluator_id=str(uuid.uuid4()), + result=_result("no", "no"), + ), + ], + ), + ] + out = compute_dataset_evaluator_results(eval_results, [evaluator]) + assert isinstance(out["intent_match::precision"].details, dict) + assert out["intent_match::precision"].details["nScored"] == 1 + + def test_source_with_no_results_produces_zeroed_report(self) -> None: + evaluator = _exact_match_evaluator( + "intent_match", + classes=["yes", "no"], + aggregators=[ + PrecisionAggregatorSpec(averaging="macro"), + ], + ) + eval_results = [ + UiPathEvalRunResult( + evaluation_name="dp1", + evaluation_run_results=[ + UiPathEvalRunResultDto( + evaluator_name="some_other_evaluator", + evaluator_id=str(uuid.uuid4()), + result=EvaluationResultDto(score=1.0), + ), + ], + ), + ] + out = compute_dataset_evaluator_results(eval_results, [evaluator]) + dto = out["intent_match::precision"] + assert dto.score == 0.0 + assert isinstance(dto.details, dict) + assert dto.details["nScored"] == 0 + + def test_duplicate_datapoint_results_are_deduped(self) -> None: + """A datapoint with two DTOs for one evaluator (e.g. a real result plus + a details-less zero from the partial-failure path, or a retry/resume + re-feed) must count once — no inflated nTotal/nSkipped, no double-count + in the matrix. The parseable DTO wins over the details-less one.""" + evaluator = _exact_match_evaluator( + "intent_match", + classes=["yes", "no"], + aggregators=[PrecisionAggregatorSpec(averaging="macro")], + ) + eval_results = [ + UiPathEvalRunResult( + evaluation_name="dp1", + evaluation_run_results=[ + UiPathEvalRunResultDto( + evaluator_name="intent_match", + evaluator_id=str(uuid.uuid4()), + result=_result("yes", "yes"), + ), + # Duplicate: details-less zero (partial-failure path shape). + UiPathEvalRunResultDto( + evaluator_name="intent_match", + evaluator_id=str(uuid.uuid4()), + result=EvaluationResultDto(score=0.0), + ), + ], + ), + ] + out = compute_dataset_evaluator_results(eval_results, [evaluator]) + details = out["intent_match::precision"].details + assert isinstance(details, dict) + # One datapoint in, one counted — not two, and the parseable one scored. + assert details["nTotal"] == 1 + assert details["nScored"] == 1 + assert details["nSkipped"] == 0 + + def test_duplicate_aggregator_type_disambiguates_by_averaging(self) -> None: + """Two aggregators of the same type get distinct keys (no overwrite).""" + evaluator = _exact_match_evaluator( + "intent_match", + classes=["yes", "no"], + aggregators=[ + PrecisionAggregatorSpec(averaging="macro"), + PrecisionAggregatorSpec(averaging="micro"), + ], + ) + eval_results = [ + UiPathEvalRunResult( + evaluation_name="dp1", + evaluation_run_results=[ + UiPathEvalRunResultDto( + evaluator_name="intent_match", + evaluator_id=str(uuid.uuid4()), + result=_result("yes", "yes"), + ), + ], + ), + ] + out = compute_dataset_evaluator_results(eval_results, [evaluator]) + # Same type appears twice → averaging suffix disambiguates so neither + # is silently overwritten. + assert set(out) == { + "intent_match::precision.macro", + "intent_match::precision.micro", + } + + def test_exact_duplicate_specs_are_deduped(self) -> None: + """Identical specs collapse to one result; duplicate confusion_matrix + (no averaging field) must not crash key disambiguation.""" + evaluator = _exact_match_evaluator( + "intent_match", + classes=["yes", "no"], + aggregators=[ + PrecisionAggregatorSpec(averaging="macro"), + PrecisionAggregatorSpec(averaging="macro"), + ConfusionMatrixAggregatorSpec(), + ConfusionMatrixAggregatorSpec(), + ], + ) + eval_results = [ + UiPathEvalRunResult( + evaluation_name="dp1", + evaluation_run_results=[ + UiPathEvalRunResultDto( + evaluator_name="intent_match", + evaluator_id=str(uuid.uuid4()), + result=_result("yes", "yes"), + ), + ], + ), + ] + out = compute_dataset_evaluator_results(eval_results, [evaluator]) + assert set(out) == { + "intent_match::precision", + "intent_match::confusion_matrix", + } + + def test_details_are_dumped_to_camelcase_wire_shape(self) -> None: + """The local path ships the same JSON shape as the platform worker: + camelCase keys, absent (not null) optional fields.""" + evaluator = _exact_match_evaluator( + "intent_match", + classes=["yes", "no"], + aggregators=[ConfusionMatrixAggregatorSpec()], + ) + eval_results = [ + UiPathEvalRunResult( + evaluation_name="dp1", + evaluation_run_results=[ + UiPathEvalRunResultDto( + evaluator_name="intent_match", + evaluator_id=str(uuid.uuid4()), + result=_result("yes", "yes"), + ), + ], + ), + ] + out = compute_dataset_evaluator_results(eval_results, [evaluator]) + details = out["intent_match::confusion_matrix"].details + assert isinstance(details, dict) + assert details["confusionMatrix"] == [[1, 0], [0, 0]] + assert details["nScored"] == 1 + # confusion_matrix variant: scalar fields are absent, not null. + assert "perClass" not in details and "macro" not in details diff --git a/packages/uipath/tests/evaluators/test_documentation_examples.py b/packages/uipath/tests/evaluators/test_documentation_examples.py index c75d94329..f9265d267 100644 --- a/packages/uipath/tests/evaluators/test_documentation_examples.py +++ b/packages/uipath/tests/evaluators/test_documentation_examples.py @@ -1,7 +1,7 @@ """Tests for examples in the eval documentation. This module ensures all code examples in the documentation actually work by -testing them with proper agent execution data. For LLM judge examples, we use +testing them with proper workload execution data. For LLM judge examples, we use mocked completions to avoid API calls. """ @@ -30,7 +30,7 @@ from uipath.eval.evaluators.tool_call_order_evaluator import ( ToolCallOrderEvaluatorJustification, ) -from uipath.eval.models import AgentExecution +from uipath.eval.models import WorkloadExecution class TestIndexExamples: @@ -39,11 +39,11 @@ class TestIndexExamples: @pytest.mark.asyncio async def test_getting_started_example(self) -> None: """Test the getting started example from index.md.""" - # Sample agent execution (this is what the docs were missing!) - agent_execution = AgentExecution( + # Sample workload execution (this is what the docs were missing!) + workload_execution = WorkloadExecution( agent_input={"query": "Greet the world"}, - agent_output={"result": "hello, world!"}, - agent_trace=[], + workload_output={"result": "hello, world!"}, + workload_trace=[], ) # Create evaluator @@ -60,7 +60,7 @@ async def test_getting_started_example(self) -> None: # Evaluate result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"expected_output": {"result": "Hello, World!"}}, ) @@ -85,16 +85,16 @@ async def test_basic_usage(self) -> None: ) ) - # agent_output must be a dict - agent_execution = AgentExecution( + # workload_output must be a dict + workload_execution = WorkloadExecution( agent_input={"query": "What is the capital of France?"}, - agent_output={"response": "The capital of France is Paris."}, - agent_trace=[], + workload_output={"response": "The capital of France is Paris."}, + workload_trace=[], ) # Evaluate - searches in the "response" field value result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"search_text": "Paris"}, ) @@ -114,15 +114,15 @@ async def test_case_sensitive_search(self) -> None: ) ) - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_output={"message": "Hello World"}, - agent_trace=[], + workload_output={"message": "Hello World"}, + workload_trace=[], ) # This will fail because of case mismatch result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"search_text": "hello"}, ) @@ -142,15 +142,15 @@ async def test_negated_search(self) -> None: ) ) - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_output={"status": "Success: Operation completed"}, - agent_trace=[], + workload_output={"status": "Success: Operation completed"}, + workload_trace=[], ) # Passes because "error" is NOT found result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"search_text": "error"}, ) @@ -169,18 +169,18 @@ async def test_target_specific_output_field(self) -> None: ) ) - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_output={ + workload_output={ "status": "success", "message": "User profile updated successfully", }, - agent_trace=[], + workload_trace=[], ) # Only searches within the "message" field result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"search_text": "updated"}, ) @@ -193,11 +193,11 @@ class TestExactMatchExamples: @pytest.mark.asyncio async def test_basic_usage(self) -> None: """Test basic usage example.""" - # agent_output must be a dict - agent_execution = AgentExecution( + # workload_output must be a dict + workload_execution = WorkloadExecution( agent_input={"query": "What is 2+2?"}, - agent_output={"result": "4"}, - agent_trace=[], + workload_output={"result": "4"}, + workload_trace=[], ) # Create evaluator - extracts "result" field for comparison @@ -214,7 +214,7 @@ async def test_basic_usage(self) -> None: # Evaluate - compares just the "result" field value result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"expected_output": {"result": "4"}}, ) @@ -223,10 +223,10 @@ async def test_basic_usage(self) -> None: @pytest.mark.asyncio async def test_case_sensitive_matching(self) -> None: """Test case-sensitive matching example.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_output={"status": "SUCCESS"}, - agent_trace=[], + workload_output={"status": "SUCCESS"}, + workload_trace=[], ) evaluator = TypeAdapter(ExactMatchEvaluator).validate_python( @@ -242,7 +242,7 @@ async def test_case_sensitive_matching(self) -> None: # Fails due to case mismatch result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"expected_output": {"status": "success"}}, ) @@ -250,7 +250,7 @@ async def test_case_sensitive_matching(self) -> None: # This would pass result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"expected_output": {"status": "SUCCESS"}}, ) @@ -259,10 +259,10 @@ async def test_case_sensitive_matching(self) -> None: @pytest.mark.asyncio async def test_matching_structured_outputs(self) -> None: """Test matching structured outputs example.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_output={"status": "success", "code": 200}, - agent_trace=[], + workload_output={"status": "success", "code": 200}, + workload_trace=[], ) evaluator = TypeAdapter(ExactMatchEvaluator).validate_python( @@ -277,7 +277,7 @@ async def test_matching_structured_outputs(self) -> None: # Entire dict structure must match result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"expected_output": {"status": "success", "code": 200}}, ) @@ -286,10 +286,10 @@ async def test_matching_structured_outputs(self) -> None: @pytest.mark.asyncio async def test_negated_mode(self) -> None: """Test negated mode example.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_output={"result": "error"}, - agent_trace=[], + workload_output={"result": "error"}, + workload_trace=[], ) evaluator = TypeAdapter(ExactMatchEvaluator).validate_python( @@ -305,7 +305,7 @@ async def test_negated_mode(self) -> None: # Passes because outputs do NOT match result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"expected_output": {"result": "success"}}, ) @@ -314,10 +314,10 @@ async def test_negated_mode(self) -> None: @pytest.mark.asyncio async def test_using_default_criteria(self) -> None: """Test using default criteria example.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_output={"status": "OK"}, - agent_trace=[], + workload_output={"status": "OK"}, + workload_trace=[], ) evaluator = TypeAdapter(ExactMatchEvaluator).validate_python( @@ -335,11 +335,57 @@ async def test_using_default_criteria(self) -> None: # Use default criteria result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, evaluation_criteria=None + workload_execution=workload_execution, evaluation_criteria=None ) assert result.score == 1.0 + @pytest.mark.asyncio + async def test_list_target_output_key(self) -> None: + """Test evaluating multiple output fields at once using a list of keys. + + Mirrors the multi-output-agent sample (list-keys-exact-match evaluator). + """ + # Agent returns a rich nested output; we only care about two summary fields. + workload_execution = WorkloadExecution( + agent_input={"customer_name": "John Doe", "items": []}, + workload_output={ + "order_id": "ORD-001", + "summary": {"status": "completed", "total": 44.97, "item_count": 3}, + "tags": ["priority", "express"], + }, + workload_trace=[], + ) + + evaluator = TypeAdapter(ExactMatchEvaluator).validate_python( + dict( + id="list-keys-exact-match", + evaluatorConfig={ + "name": "ListKeysExactMatch", + # Pass a list to assert multiple fields in a single evaluator run. + "target_output_key": ["summary.status", "summary.total"], + }, + ) + ) + + # Both keys match → score 1.0 + result = await evaluator.validate_and_evaluate_criteria( + workload_execution=workload_execution, + evaluation_criteria={ + "expected_output": {"summary": {"status": "completed", "total": 44.97}} + }, + ) + assert result.score == 1.0 + + # One key differs → score 0.0 + result = await evaluator.validate_and_evaluate_criteria( + workload_execution=workload_execution, + evaluation_criteria={ + "expected_output": {"summary": {"status": "completed", "total": 999.0}} + }, + ) + assert result.score == 0.0 + class TestJsonSimilarityExamples: """Test examples from docs/eval/json_similarity.md.""" @@ -347,10 +393,10 @@ class TestJsonSimilarityExamples: @pytest.mark.asyncio async def test_basic_json_comparison(self) -> None: """Test basic JSON comparison example.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_output={"name": "John Doe", "age": 30, "city": "New York"}, - agent_trace=[], + workload_output={"name": "John Doe", "age": 30, "city": "New York"}, + workload_trace=[], ) evaluator = TypeAdapter(JsonSimilarityEvaluator).validate_python( @@ -364,7 +410,7 @@ async def test_basic_json_comparison(self) -> None: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_output": {"name": "John Doe", "age": 30, "city": "New York"} }, @@ -379,10 +425,10 @@ async def test_basic_json_comparison(self) -> None: @pytest.mark.asyncio async def test_numeric_tolerance(self) -> None: """Test numeric tolerance example.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_output={"temperature": 20.5, "humidity": 65}, - agent_trace=[], + workload_output={"temperature": 20.5, "humidity": 65}, + workload_trace=[], ) evaluator = TypeAdapter(JsonSimilarityEvaluator).validate_python( @@ -394,7 +440,7 @@ async def test_numeric_tolerance(self) -> None: # Slightly different numbers result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_output": {"temperature": 20.3, "humidity": 65} }, @@ -406,10 +452,10 @@ async def test_numeric_tolerance(self) -> None: @pytest.mark.asyncio async def test_string_similarity(self) -> None: """Test string similarity example.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_output={"status": "completed successfully"}, - agent_trace=[], + workload_output={"status": "completed successfully"}, + workload_trace=[], ) evaluator = TypeAdapter(JsonSimilarityEvaluator).validate_python( @@ -421,7 +467,7 @@ async def test_string_similarity(self) -> None: # Similar but not exact string result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_output": {"status": "completed sucessfully"} # typo }, @@ -434,13 +480,13 @@ async def test_string_similarity(self) -> None: @pytest.mark.asyncio async def test_nested_structures(self) -> None: """Test nested structures example.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_output={ + workload_output={ "user": {"name": "Alice", "profile": {"age": 25, "location": "Paris"}}, "status": "active", }, - agent_trace=[], + workload_trace=[], ) evaluator = TypeAdapter(JsonSimilarityEvaluator).validate_python( @@ -451,7 +497,7 @@ async def test_nested_structures(self) -> None: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_output": { "user": { @@ -468,10 +514,10 @@ async def test_nested_structures(self) -> None: @pytest.mark.asyncio async def test_array_comparison(self) -> None: """Test array comparison example.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_output={"items": ["apple", "banana", "orange"]}, - agent_trace=[], + workload_output={"items": ["apple", "banana", "orange"]}, + workload_trace=[], ) evaluator = TypeAdapter(JsonSimilarityEvaluator).validate_python( @@ -483,7 +529,7 @@ async def test_array_comparison(self) -> None: # Partial match (2 out of 3 correct) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_output": {"items": ["apple", "banana", "grape"]} }, @@ -496,14 +542,14 @@ async def test_array_comparison(self) -> None: @pytest.mark.asyncio async def test_handling_extra_keys(self) -> None: """Test handling extra keys in actual output example.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_output={ + workload_output={ "name": "Bob", "age": 30, "extra_field": "ignored", # Extra field in actual output }, - agent_trace=[], + workload_trace=[], ) evaluator = TypeAdapter(JsonSimilarityEvaluator).validate_python( @@ -515,7 +561,7 @@ async def test_handling_extra_keys(self) -> None: # Only expected keys are evaluated result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"expected_output": {"name": "Bob", "age": 30}}, ) @@ -524,13 +570,13 @@ async def test_handling_extra_keys(self) -> None: @pytest.mark.asyncio async def test_target_specific_field(self) -> None: """Test target specific field example.""" - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_output={ + workload_output={ "result": {"score": 95, "passed": True}, "metadata": {"timestamp": "2024-01-01"}, }, - agent_trace=[], + workload_trace=[], ) evaluator = TypeAdapter(JsonSimilarityEvaluator).validate_python( @@ -545,7 +591,7 @@ async def test_target_specific_field(self) -> None: # Only compares the "result" field result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_output": {"result": {"score": 95, "passed": True}} }, @@ -579,10 +625,10 @@ async def test_basic_semantic_similarity(self, mocker: MockerFixture) -> None: async def mock_chat_completions(*args: Any, **kwargs: Any) -> Any: return mock_response - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"query": "What is the capital of France?"}, - agent_output={"answer": "Paris is the capital city of France."}, - agent_trace=[], + workload_output={"answer": "Paris is the capital city of France."}, + workload_trace=[], ) evaluator = TypeAdapter(LLMJudgeOutputEvaluator).validate_python( @@ -599,7 +645,7 @@ async def mock_chat_completions(*args: Any, **kwargs: Any) -> Any: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_output": {"answer": "The capital of France is Paris."} }, @@ -639,12 +685,12 @@ async def mock_chat_completions(*args: Any, **kwargs: Any) -> Any: Provide a score from 0-100 based on semantic similarity. """ - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_output={ + workload_output={ "message": "The product has been successfully added to your cart." }, - agent_trace=[], + workload_trace=[], ) evaluator = TypeAdapter(LLMJudgeOutputEvaluator).validate_python( @@ -662,7 +708,7 @@ async def mock_chat_completions(*args: Any, **kwargs: Any) -> Any: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_output": {"message": "Item added to shopping cart."} }, @@ -694,9 +740,9 @@ async def test_evaluating_natural_language_quality( async def mock_chat_completions(*args: Any, **kwargs: Any) -> Any: return mock_response - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"task": "Write a professional email"}, - agent_output={ + workload_output={ "email": """Dear Customer, Thank you for your inquiry. We have reviewed your request @@ -706,7 +752,7 @@ async def mock_chat_completions(*args: Any, **kwargs: Any) -> Any: Best regards, Support Team""" }, - agent_trace=[], + workload_trace=[], ) evaluator = TypeAdapter(LLMJudgeOutputEvaluator).validate_python( @@ -723,7 +769,7 @@ async def mock_chat_completions(*args: Any, **kwargs: Any) -> Any: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_output": { "email": "A professional, courteous response addressing the customer's inquiry" @@ -756,15 +802,15 @@ async def test_strict_json_similarity(self, mocker: MockerFixture) -> None: async def mock_chat_completions(*args: Any, **kwargs: Any) -> Any: return mock_response - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={}, - agent_output={ + workload_output={ "status": "success", "user_id": 12345, "name": "John Doe", "email": "john@example.com", }, - agent_trace=[], + workload_trace=[], ) evaluator = TypeAdapter( @@ -782,7 +828,7 @@ async def mock_chat_completions(*args: Any, **kwargs: Any) -> Any: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_output": { "status": "success", @@ -821,10 +867,10 @@ async def test_basic_trajectory_evaluation(self, mocker: MockerFixture) -> None: async def mock_chat_completions(*args: Any, **kwargs: Any) -> Any: return mock_response - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"user_query": "Book a flight to Paris"}, - agent_output={"booking_id": "FL123", "status": "confirmed"}, - agent_trace=[ + workload_output={"booking_id": "FL123", "status": "confirmed"}, + workload_trace=[ # Trace contains spans showing the agent's execution path # Each span represents a step in the agent's decision-making ], @@ -843,7 +889,7 @@ async def mock_chat_completions(*args: Any, **kwargs: Any) -> Any: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_agent_behavior": """ The agent should: @@ -879,10 +925,10 @@ async def test_validating_tool_usage_sequence(self, mocker: MockerFixture) -> No async def mock_chat_completions(*args: Any, **kwargs: Any) -> Any: return mock_response - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"task": "Update user profile and send notification"}, - agent_output={"status": "completed"}, - agent_trace=[ + workload_output={"status": "completed"}, + workload_trace=[ # Spans showing: validate_user -> update_profile -> send_notification ], ) @@ -900,7 +946,7 @@ async def mock_chat_completions(*args: Any, **kwargs: Any) -> Any: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_agent_behavior": """ The agent must: @@ -937,10 +983,10 @@ async def test_trajectory_simulation(self, mocker: MockerFixture) -> None: async def mock_chat_completions(*args: Any, **kwargs: Any) -> Any: return mock_response - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"query": "Book a flight to Paris for tomorrow"}, - agent_output={"booking_id": "FL123", "status": "confirmed"}, - agent_trace=[ + workload_output={"booking_id": "FL123", "status": "confirmed"}, + workload_trace=[ # Execution spans showing tool calls and their simulated responses ], simulation_instructions=""" @@ -965,7 +1011,7 @@ async def mock_chat_completions(*args: Any, **kwargs: Any) -> Any: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "expected_agent_behavior": """ The agent should: @@ -996,7 +1042,7 @@ async def test_basic_tool_call_order(self) -> None: """Test basic tool call order validation example with sample trace.""" from opentelemetry.sdk.trace import ReadableSpan - # Sample agent execution with tool calls in trace (this is what was missing in docs!) + # Sample workload execution with tool calls in trace (this is what was missing in docs!) mock_spans = [ ReadableSpan( name="validate_user", @@ -1018,10 +1064,10 @@ async def test_basic_tool_call_order(self) -> None: ), ] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"task": "Process user order"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = TypeAdapter(ToolCallOrderEvaluator).validate_python( @@ -1032,7 +1078,7 @@ async def test_basic_tool_call_order(self) -> None: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls_order": ["validate_user", "check_inventory", "create_order"] }, @@ -1067,10 +1113,10 @@ async def test_strict_order_validation(self) -> None: ), ] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"task": "Access secured resource"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = TypeAdapter(ToolCallOrderEvaluator).validate_python( @@ -1081,7 +1127,7 @@ async def test_strict_order_validation(self) -> None: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls_order": [ "authenticate_user", @@ -1121,10 +1167,10 @@ async def test_partial_credit_with_lcs(self) -> None: ), ] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"task": "Search and display"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = TypeAdapter(ToolCallOrderEvaluator).validate_python( @@ -1138,7 +1184,7 @@ async def test_partial_credit_with_lcs(self) -> None: expected = ["search", "filter", "sort", "display"] result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={"tool_calls_order": expected}, ) @@ -1179,10 +1225,10 @@ async def test_database_transaction_sequence(self) -> None: ), ] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"task": "Update database"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = TypeAdapter(ToolCallOrderEvaluator).validate_python( @@ -1193,7 +1239,7 @@ async def test_database_transaction_sequence(self) -> None: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls_order": [ "begin_transaction", @@ -1245,10 +1291,10 @@ async def test_api_integration_workflow(self) -> None: ), ] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"task": "API integration"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = TypeAdapter(ToolCallOrderEvaluator).validate_python( @@ -1259,7 +1305,7 @@ async def test_api_integration_workflow(self) -> None: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls_order": [ "get_api_token", @@ -1299,10 +1345,10 @@ async def test_using_default_criteria(self) -> None: ), ] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"task": "Standard workflow"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = TypeAdapter(ToolCallOrderEvaluator).validate_python( @@ -1320,7 +1366,7 @@ async def test_using_default_criteria(self) -> None: # Use default criteria result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, evaluation_criteria=None + workload_execution=workload_execution, evaluation_criteria=None ) assert result.score == 1.0 @@ -1334,7 +1380,7 @@ async def test_basic_count_validation(self) -> None: """Test basic count validation example with sample trace.""" from opentelemetry.sdk.trace import ReadableSpan - # Sample agent execution with tool calls (this is what was missing in docs!) + # Sample workload execution with tool calls (this is what was missing in docs!) mock_spans = [ ReadableSpan( name="fetch_data", @@ -1380,10 +1426,10 @@ async def test_basic_count_validation(self) -> None: ), ] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"task": "Fetch and process data"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = TypeAdapter(ToolCallCountEvaluator).validate_python( @@ -1394,7 +1440,7 @@ async def test_basic_count_validation(self) -> None: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls_count": { "fetch_data": ("=", 1), @@ -1438,10 +1484,10 @@ async def test_using_comparison_operators(self) -> None: ), ] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"task": "API operation"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = TypeAdapter(ToolCallCountEvaluator).validate_python( @@ -1452,7 +1498,7 @@ async def test_using_comparison_operators(self) -> None: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls_count": { "validate": (">=", 1), # At least once @@ -1491,10 +1537,10 @@ async def test_strict_mode_all_or_nothing(self) -> None: ), ] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"task": "Database operation"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = TypeAdapter(ToolCallCountEvaluator).validate_python( @@ -1505,7 +1551,7 @@ async def test_strict_mode_all_or_nothing(self) -> None: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls_count": { "authenticate": ("=", 1), @@ -1551,10 +1597,10 @@ async def test_preventing_redundant_calls(self) -> None: ), ] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"task": "Optimize resource usage"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = TypeAdapter(ToolCallCountEvaluator).validate_python( @@ -1566,7 +1612,7 @@ async def test_preventing_redundant_calls(self) -> None: # Ensure expensive operations aren't called too many times result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls_count": { "expensive_api_call": ( @@ -1612,10 +1658,10 @@ async def test_loop_validation(self) -> None: ] ) - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"task": "Process 10 items"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = TypeAdapter(ToolCallCountEvaluator).validate_python( @@ -1627,7 +1673,7 @@ async def test_loop_validation(self) -> None: # Verify loop processed correct number of items result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls_count": { "process_item": ("=", 10), # Should process 10 items @@ -1648,7 +1694,7 @@ async def test_basic_argument_validation(self) -> None: """Test basic argument validation example with sample trace.""" from opentelemetry.sdk.trace import ReadableSpan - # Sample agent execution with tool calls and arguments (this is what was missing in docs!) + # Sample workload execution with tool calls and arguments (this is what was missing in docs!) mock_spans = [ ReadableSpan( name="update_user", @@ -1661,10 +1707,10 @@ async def test_basic_argument_validation(self) -> None: ), ] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"user_id": 123, "action": "update"}, - agent_output={"status": "success"}, - agent_trace=mock_spans, + workload_output={"status": "success"}, + workload_trace=mock_spans, ) evaluator = TypeAdapter(ToolCallArgsEvaluator).validate_python( @@ -1679,7 +1725,7 @@ async def test_basic_argument_validation(self) -> None: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls": [ { @@ -1713,10 +1759,10 @@ async def test_strict_mode_exact_matching(self) -> None: ), ] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"task": "Send email"}, - agent_output={"status": "sent"}, - agent_trace=mock_spans, + workload_output={"status": "sent"}, + workload_trace=mock_spans, ) evaluator = TypeAdapter(ToolCallArgsEvaluator).validate_python( @@ -1731,7 +1777,7 @@ async def test_strict_mode_exact_matching(self) -> None: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls": [ { @@ -1765,10 +1811,10 @@ async def test_subset_mode_partial_validation(self) -> None: ), ] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"task": "Create user"}, - agent_output={"status": "created"}, - agent_trace=mock_spans, + workload_output={"status": "created"}, + workload_trace=mock_spans, ) evaluator = TypeAdapter(ToolCallArgsEvaluator).validate_python( @@ -1784,7 +1830,7 @@ async def test_subset_mode_partial_validation(self) -> None: # Only validate critical fields result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls": [ { @@ -1832,10 +1878,10 @@ async def test_multiple_tool_calls(self) -> None: ), ] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"task": "Data pipeline"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = TypeAdapter(ToolCallArgsEvaluator).validate_python( @@ -1850,7 +1896,7 @@ async def test_multiple_tool_calls(self) -> None: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls": [ { @@ -1888,10 +1934,10 @@ async def test_nested_arguments(self) -> None: ), ] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"task": "evaluatorConfigure API service"}, - agent_output={"status": "evaluatorConfigured"}, - agent_trace=mock_spans, + workload_output={"status": "evaluatorConfigured"}, + workload_trace=mock_spans, ) evaluator = TypeAdapter(ToolCallArgsEvaluator).validate_python( @@ -1906,7 +1952,7 @@ async def test_nested_arguments(self) -> None: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls": [ { @@ -1962,10 +2008,10 @@ async def test_non_strict_proportional_scoring(self) -> None: ), ] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"task": "Update user profile"}, - agent_output={"status": "updated"}, - agent_trace=mock_spans, + workload_output={"status": "updated"}, + workload_trace=mock_spans, ) evaluator = TypeAdapter(ToolCallArgsEvaluator).validate_python( @@ -1980,7 +2026,7 @@ async def test_non_strict_proportional_scoring(self) -> None: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_calls": [ { @@ -2014,7 +2060,7 @@ async def test_basic_output_validation(self) -> None: """Test basic output validation example with sample trace.""" from opentelemetry.sdk.trace import ReadableSpan - # Sample agent execution with tool calls and outputs (this is what was missing in docs!) + # Sample workload execution with tool calls and outputs (this is what was missing in docs!) mock_spans = [ ReadableSpan( name="get_user", @@ -2027,10 +2073,10 @@ async def test_basic_output_validation(self) -> None: ), ] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"user_id": 123}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = TypeAdapter(ToolCallOutputEvaluator).validate_python( @@ -2041,7 +2087,7 @@ async def test_basic_output_validation(self) -> None: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_outputs": [ { @@ -2071,10 +2117,10 @@ async def test_strict_mode_exact_output_matching(self) -> None: ), ] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"operation": "multiply"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = TypeAdapter(ToolCallOutputEvaluator).validate_python( @@ -2085,7 +2131,7 @@ async def test_strict_mode_exact_output_matching(self) -> None: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_outputs": [ { @@ -2124,10 +2170,10 @@ async def test_multiple_tool_outputs(self) -> None: ), ] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"task": "Process items"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = TypeAdapter(ToolCallOutputEvaluator).validate_python( @@ -2138,7 +2184,7 @@ async def test_multiple_tool_outputs(self) -> None: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_outputs": [ {"name": "fetch", "output": '{"data": ["item1", "item2"]}'}, @@ -2166,10 +2212,10 @@ async def test_error_handling_validation(self) -> None: ), ] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"email": "invalid-email"}, - agent_output={"status": "validation_failed"}, - agent_trace=mock_spans, + workload_output={"status": "validation_failed"}, + workload_trace=mock_spans, ) evaluator = TypeAdapter(ToolCallOutputEvaluator).validate_python( @@ -2180,7 +2226,7 @@ async def test_error_handling_validation(self) -> None: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_outputs": [ { @@ -2229,10 +2275,10 @@ async def test_non_strict_proportional_scoring(self) -> None: ), ] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"task": "Process data pipeline"}, - agent_output={"status": "completed"}, - agent_trace=mock_spans, + workload_output={"status": "completed"}, + workload_trace=mock_spans, ) evaluator = TypeAdapter(ToolCallOutputEvaluator).validate_python( @@ -2246,7 +2292,7 @@ async def test_non_strict_proportional_scoring(self) -> None: ) result = await evaluator.validate_and_evaluate_criteria( - agent_execution=agent_execution, + workload_execution=workload_execution, evaluation_criteria={ "tool_outputs": [ { diff --git a/packages/uipath/tests/evaluators/test_eval_level_expected_output.py b/packages/uipath/tests/evaluators/test_eval_level_expected_output.py index 1c2b8fd44..c161cbfc6 100644 --- a/packages/uipath/tests/evaluators/test_eval_level_expected_output.py +++ b/packages/uipath/tests/evaluators/test_eval_level_expected_output.py @@ -28,7 +28,7 @@ EvaluationItem, EvaluationSet, ) -from uipath.eval.models.models import AgentExecution +from uipath.eval.models.models import WorkloadExecution # ───────────────────────────────────────────────────────────────── # Model Tests @@ -387,10 +387,10 @@ class TestExactMatchWithEvaluationLevelExpectedOutput: @pytest.mark.asyncio async def test_exact_match_with_evaluation_level_expected_output(self) -> None: """ExactMatchEvaluator uses evaluation-level expectedOutput when criteria is null.""" - execution = AgentExecution( + execution = WorkloadExecution( agent_input={"query": "2+2"}, - agent_output={"result": 4}, - agent_trace=[], + workload_output={"result": 4}, + workload_trace=[], ) evaluator = ExactMatchEvaluator.model_validate( {"evaluatorConfig": {"name": "Test"}, "id": str(uuid.uuid4())} @@ -408,10 +408,10 @@ async def test_exact_match_with_evaluation_level_expected_output(self) -> None: @pytest.mark.asyncio async def test_exact_match_per_evaluator_overrides_evaluation_level(self) -> None: """Per-evaluator expectedOutput overrides evaluation-level.""" - execution = AgentExecution( + execution = WorkloadExecution( agent_input={"query": "2+2"}, - agent_output={"result": 4}, - agent_trace=[], + workload_output={"result": 4}, + workload_trace=[], ) evaluator = ExactMatchEvaluator.model_validate( {"evaluatorConfig": {"name": "Test"}, "id": str(uuid.uuid4())} @@ -433,10 +433,10 @@ class TestJsonSimilarityWithEvaluationLevelExpectedOutput: @pytest.mark.asyncio async def test_json_similarity_with_evaluation_level_expected_output(self) -> None: """JsonSimilarityEvaluator uses evaluation-level expectedOutput.""" - execution = AgentExecution( + execution = WorkloadExecution( agent_input={"input": "Test"}, - agent_output={"name": "John", "age": 30, "city": "NYC"}, - agent_trace=[], + workload_output={"name": "John", "age": 30, "city": "NYC"}, + workload_trace=[], ) evaluator = JsonSimilarityEvaluator.model_validate( {"evaluatorConfig": {"name": "Test"}, "id": str(uuid.uuid4())} @@ -490,10 +490,10 @@ async def mock_chat_completions(*args: Any, **kwargs: Any) -> Any: } ) - execution = AgentExecution( + execution = WorkloadExecution( agent_input={"query": "test"}, - agent_output={"result": "test output"}, - agent_trace=[], + workload_output={"result": "test output"}, + workload_trace=[], ) # Criteria built from evaluation-level expectedOutput @@ -627,10 +627,10 @@ async def test_e2e_exact_match_null_criteria_with_evaluation_level(self) -> None "evaluationCriterias": {evaluator_id: None}, } ) - execution = AgentExecution( + execution = WorkloadExecution( agent_input={"query": "2+2"}, - agent_output={"result": 4}, - agent_trace=[], + workload_output={"result": 4}, + workload_trace=[], ) # Simulate runtime merge @@ -680,10 +680,10 @@ async def test_e2e_mixed_evaluators_with_evaluation_level(self) -> None: }, } ) - execution = AgentExecution( + execution = WorkloadExecution( agent_input={"query": "hello"}, - agent_output="Hello World", - agent_trace=[], + workload_output="Hello World", + workload_trace=[], ) # Process exact-match (output-based) @@ -729,10 +729,10 @@ async def test_e2e_per_evaluator_override_with_evaluation_level(self) -> None: }, } ) - execution = AgentExecution( + execution = WorkloadExecution( agent_input={"query": "2+2"}, - agent_output={"result": 4}, - agent_trace=[], + workload_output={"result": 4}, + workload_trace=[], ) # Simulate runtime merge diff --git a/packages/uipath/tests/evaluators/test_evaluator_helpers.py b/packages/uipath/tests/evaluators/test_evaluator_helpers.py index 84eb1159f..6064381cb 100644 --- a/packages/uipath/tests/evaluators/test_evaluator_helpers.py +++ b/packages/uipath/tests/evaluators/test_evaluator_helpers.py @@ -12,6 +12,9 @@ import pytest from uipath.eval._helpers.evaluators_helpers import ( + _calls_match, + _match_key, + _sanitize_tool_name, extract_tool_calls, extract_tool_calls_names, extract_tool_calls_outputs, @@ -681,6 +684,54 @@ def test_extract_tool_calls_outputs_filters_non_tool_spans( assert "non_tool_span" not in output_names assert len(result) == 3 + def test_extractors_skip_synthesized_tool_spans(self) -> None: + """Spans tagged with tool.synthesized=True (BPMN container spans + synthesized for trajectory rendering) must be filtered from all three + per-call extractors so they don't pollute tool-call evaluator actuals. + """ + from opentelemetry.sdk.trace import ReadableSpan + + synth_process = ReadableSpan( + name="Instance: abc", + start_time=0, + end_time=1, + attributes={ + "tool.name": "FlowExecution", + "tool.synthesized": True, + "input.value": '{"instanceId": "abc"}', + "output.value": "Status: Completed", + }, + ) + synth_element = ReadableSpan( + name="Autonomous Agent", + start_time=2, + end_time=3, + attributes={ + "tool.name": "ServiceTask: Autonomous Agent", + "tool.synthesized": True, + "input.value": "{}", + "output.value": "Status: Completed", + }, + ) + real_tool = ReadableSpan( + name="Tool call - web_search", + start_time=4, + end_time=5, + attributes={ + "tool.name": "web_search", + "input.value": '{"query": "x"}', + "output.value": '{"content": "ok"}', + }, + ) + + spans = [synth_process, synth_element, real_tool] + + assert extract_tool_calls_names(spans) == ["web_search"] + calls = extract_tool_calls(spans) + assert [c.name for c in calls] == ["web_search"] + outputs = extract_tool_calls_outputs(spans) + assert [o.name for o in outputs] == ["web_search"] + def test_all_extraction_functions_consistent(self, sample_spans: list[Any]) -> None: """Test that all extraction functions return consistent results.""" names = extract_tool_calls_names(sample_spans) @@ -820,3 +871,330 @@ def test_extract_tool_calls_outputs_with_json_non_dict_value(self) -> None: assert result[0].name == "json_array_tool" # Should use the original string when parsed JSON is not a dict assert result[0].output == '["item1", "item2", "item3"]' + + +class TestIdAwareExtraction: + """Verify tool.id propagation through the three extractors, plus the + include_args=False optimization and the JSON-first parse fallback. + """ + + def test_extractors_read_tool_id_when_present(self) -> None: + """When a span carries `tool.id`, it must surface on ToolCall/ToolOutput.id.""" + from opentelemetry.sdk.trace import ReadableSpan + + span = ReadableSpan( + name="Tool call - Web_Search", + start_time=0, + end_time=1, + attributes={ + "tool.name": "Web_Search", + "tool.id": "7abae702-f898-4cc9-95f1-c365b9a857f9", + "input.value": "{}", + "output.value": '{"content": "ok"}', + }, + ) + calls = extract_tool_calls([span]) + outputs = extract_tool_calls_outputs([span]) + assert calls[0].id == "7abae702-f898-4cc9-95f1-c365b9a857f9" + assert calls[0].name == "Web_Search" + assert outputs[0].id == "7abae702-f898-4cc9-95f1-c365b9a857f9" + assert outputs[0].name == "Web_Search" + + def test_extractors_preserve_falsy_but_present_tool_id(self) -> None: + """tool.id of 0 or empty string is unusual but legal — must not be silently dropped. + + Original code used `if tool_id` which would treat 0 / '' as missing. + Fix uses `is not None`. + """ + from opentelemetry.sdk.trace import ReadableSpan + + for falsy_id in (0, "", False): + span = ReadableSpan( + name="t", + start_time=0, + end_time=1, + attributes={ + "tool.name": "f", + "tool.id": falsy_id, + "input.value": "{}", + "output.value": '{"content": "ok"}', + }, + ) + calls = extract_tool_calls([span]) + outputs = extract_tool_calls_outputs([span]) + assert calls[0].id == str(falsy_id), f"falsy id {falsy_id!r} was dropped" + assert outputs[0].id == str(falsy_id), f"falsy id {falsy_id!r} was dropped" + + def test_extract_tool_calls_parses_json_literals(self) -> None: + """input.value with JSON `true`/`false`/`null` should parse cleanly. + + `ast.literal_eval` doesn't recognise those tokens (Python uses + True/False/None); the extractor now tries `json.loads` first and only + falls back to `ast.literal_eval` on JSON parse failure. + """ + from opentelemetry.sdk.trace import ReadableSpan + + span = ReadableSpan( + name="t", + start_time=0, + end_time=1, + attributes={ + "tool.name": "t", + "input.value": '{"a": true, "b": false, "c": null}', + }, + ) + calls = extract_tool_calls([span]) + assert calls[0].args == {"a": True, "b": False, "c": None} + + def test_extract_tool_calls_falls_back_to_python_literal(self) -> None: + """Single-quoted Python dict repr (the historical input shape) still parses.""" + from opentelemetry.sdk.trace import ReadableSpan + + span = ReadableSpan( + name="t", + start_time=0, + end_time=1, + attributes={ + "tool.name": "t", + "input.value": "{'a': 1, 'b': 'two'}", # JSON-invalid, Python-valid + }, + ) + calls = extract_tool_calls([span]) + assert calls[0].args == {"a": 1, "b": "two"} + + def test_extract_tool_calls_non_dict_parsed_result_yields_empty_args(self) -> None: + """If input.value parses to a non-dict (e.g. a bare string), args→{}. + + Avoids pydantic validation failures from feeding a non-dict into + ToolCall(args=...). + """ + from opentelemetry.sdk.trace import ReadableSpan + + span = ReadableSpan( + name="t", + start_time=0, + end_time=1, + attributes={ + "tool.name": "t", + "input.value": '"hello"', # JSON-valid string, not a dict + }, + ) + calls = extract_tool_calls([span]) + assert calls[0].args == {} + + def test_extract_tool_calls_include_args_false_skips_parse(self) -> None: + """With include_args=False, broken input.value is not parsed and doesn't raise. + + Used by count / order evaluators that don't need args. + """ + from opentelemetry.sdk.trace import ReadableSpan + + span = ReadableSpan( + name="t", + start_time=0, + end_time=1, + attributes={ + "tool.name": "t", + "tool.id": "abc", + "input.value": "this is not valid python or json{{{", + }, + ) + calls = extract_tool_calls([span], include_args=False) + assert len(calls) == 1 + assert calls[0].name == "t" + assert calls[0].id == "abc" + assert calls[0].args == {} # short-circuited, not parsed + + def test_extractors_default_id_to_none_when_absent(self) -> None: + """Spans without `tool.id` produce ToolCall/ToolOutput with id=None (back-compat).""" + from opentelemetry.sdk.trace import ReadableSpan + + span = ReadableSpan( + name="Tool call - legacy", + start_time=0, + end_time=1, + attributes={ + "tool.name": "legacy_tool", + "input.value": "{}", + "output.value": '{"content": "ok"}', + }, + ) + calls = extract_tool_calls([span]) + outputs = extract_tool_calls_outputs([span]) + assert calls[0].id is None + assert outputs[0].id is None + + +class TestIdAwareMatching: + """Verify id-aware matching across all four tool-call scoring functions. + + For each function: an Expected criterion authored against the tool's id + matches the actual call when the actual carries the same id, even if the + `name` differs (the common case after a tool rename or the + 'Web Search' → 'Web_Search' display-vs-runtime divergence). + """ + + def test_args_score_matches_by_id_when_names_differ(self) -> None: + """Expected keyed by id matches actual with same id but different name.""" + from uipath.eval._helpers.evaluators_helpers import tool_calls_args_score + + actual = [ToolCall(name="Web_Search", id="uuid-1", args={"q": "x"})] + expected = [ToolCall(name="Web Search", id="uuid-1", args={"q": "x"})] + score, _ = tool_calls_args_score(actual, expected) + assert score == 1.0 + + def test_args_score_strict_kind_no_id_fallback(self) -> None: + """Strict kind: actual has id → id-only mode, no name fallback even when actual.name matches expected.name.""" + from uipath.eval._helpers.evaluators_helpers import tool_calls_args_score + + actual = [ToolCall(name="Web_Search", id="uuid-1", args={"q": "x"})] + expected = [ToolCall(name="Web_Search", args={"q": "x"})] + score, _ = tool_calls_args_score(actual, expected) + assert score == 0.0 # actual.id="uuid-1" != expected.name="Web_Search" + + def test_args_score_name_only_when_actual_has_no_id(self) -> None: + """When actual has no id, sanitised-name comparison is the only path.""" + from uipath.eval._helpers.evaluators_helpers import tool_calls_args_score + + actual = [ToolCall(name="Web_Search", args={"q": "x"})] + expected = [ToolCall(name="Web Search", args={"q": "x"})] + score, _ = tool_calls_args_score(actual, expected) + assert score == 1.0 + + def test_args_score_no_match_when_ids_differ(self) -> None: + """Different ids → no match even with same name.""" + from uipath.eval._helpers.evaluators_helpers import tool_calls_args_score + + actual = [ToolCall(name="Web_Search", id="uuid-A", args={"q": "x"})] + expected = [ToolCall(name="Web_Search", id="uuid-B", args={"q": "x"})] + score, _ = tool_calls_args_score(actual, expected) + assert score == 0.0 + + def test_output_score_matches_by_id(self) -> None: + from uipath.eval._helpers.evaluators_helpers import tool_calls_output_score + + actual = [ToolOutput(name="Web_Search", id="uuid-1", output="ok")] + expected = [ToolOutput(name="Web Search", id="uuid-1", output="ok")] + score, _ = tool_calls_output_score(actual, expected) + assert score == 1.0 + + def test_count_by_name_and_id_helper(self) -> None: + """Strict per-call kind: id-keyed when call has id, name-keyed otherwise — never both.""" + from uipath.eval._helpers.evaluators_helpers import ( + count_tool_calls_by_name_and_id, + ) + + calls = [ + ToolCall(name="Web_Search", id="uuid-1", args={}), + ToolCall(name="Web_Search", id="uuid-1", args={}), + ToolCall(name="get_temp", args={}), # no id + ] + counts = count_tool_calls_by_name_and_id(calls) + assert counts == {"uuid-1": 2, "get_temp": 1} + # Name key is NOT populated when id is present — kind separation. + assert "Web_Search" not in counts + + def test_order_score_with_ids_matches_id_keyed_expected(self) -> None: + """Strict kind: actual has id → only id-keyed expected matches; legacy name-keyed against id-bearing actual is a miss.""" + from uipath.eval._helpers.evaluators_helpers import ( + tool_calls_order_score_with_ids, + ) + + actual = [ + ToolCall(name="Web_Search", id="uuid-1", args={}), + ToolCall(name="Web_Search", id="uuid-1", args={}), + ] + # Expected authored by id matches. + score, _ = tool_calls_order_score_with_ids(actual, ["uuid-1", "uuid-1"]) + assert score == 1.0 + # Expected authored by name against id-bearing actual is a miss (no cross-kind). + score, _ = tool_calls_order_score_with_ids(actual, ["Web_Search", "Web_Search"]) + assert score == 0.0 + # Mixed expected: only the id-keyed element matches. + score, _ = tool_calls_order_score_with_ids(actual, ["uuid-1", "Web_Search"]) + assert 0.0 < score < 1.0 + + def test_order_score_with_ids_back_compat_when_id_absent(self) -> None: + """When actual has no ids (legacy traces), comparison is name-only.""" + from uipath.eval._helpers.evaluators_helpers import ( + tool_calls_order_score_with_ids, + ) + + actual = [ + ToolCall(name="get_temp", args={}), + ToolCall(name="get_humidity", args={}), + ] + score, _ = tool_calls_order_score_with_ids(actual, ["get_temp", "get_humidity"]) + assert score == 1.0 + + +class TestSanitizedNameMatch: + """Sanitised-name fallback in ``_match_key`` / ``_calls_match`` — id-equality wins first.""" + + @staticmethod + def _reference_sanitize(name: str) -> str: + """Pinned copy of ``uipath_langchain.agent.tools.utils.sanitize_tool_name``.""" + import re + + trim_whitespaces = "_".join(name.split()) + sanitized = re.sub(r"[^a-zA-Z0-9_-]", "", trim_whitespaces) + return sanitized[:64] + + @pytest.mark.parametrize( + "raw", + [ + "Web Search", + "Google Sheets / Read", + "Add Numbers", + "tool with spaces and (parens)", + "snake_case_tool", + "kebab-case-tool", + "alreadySanitised", + " multiple whitespace ", + "very-long-name-" + "x" * 100, + "", + ], + ) + def test_normalize_matches_langchain_reference(self, raw: str) -> None: + assert _sanitize_tool_name(raw) == self._reference_sanitize(raw) + + def test_normalize_handles_none(self) -> None: + assert _sanitize_tool_name(None) == "" + + def test_match_key_display_vs_sanitised(self) -> None: + assert _match_key("Web_Search", None, "Web Search") is True + + def test_match_key_id_wins_when_present(self) -> None: + assert _match_key("Web_Search", "webSearch1", "webSearch1") is True + # Strict kind: actual has id → display-name expected is rejected (no cross-kind). + assert _match_key("Web_Search", "webSearch1", "Web Search") is False + + def test_match_key_mismatch_after_sanitising(self) -> None: + assert _match_key("Web_Search", None, "Image_Search") is False + + def test_calls_match_display_vs_sanitised(self) -> None: + actual = ToolCall(name="Web_Search", args={}) + expected = ToolCall(name="Web Search", args={}) + assert _calls_match(actual, expected) is True + + def test_calls_match_id_equality_unchanged(self) -> None: + actual = ToolCall(name="Web_Search", id="webSearch1", args={}) + expected = ToolCall(name="totally different", id="webSearch1", args={}) + assert _calls_match(actual, expected) is True + + def test_calls_match_output_display_vs_sanitised(self) -> None: + actual = ToolOutput(name="Web_Search", output="x") + expected = ToolOutput(name="Web Search", output="x") + assert _calls_match(actual, expected) is True + + def test_count_score_display_name_matches_sanitised_actual(self) -> None: + actual = {"Web_Search": 2} + expected = {"Web Search": ("==", 2)} + score, _ = tool_calls_count_score(actual, expected) + assert score == 1.0 + + def test_count_score_id_keyed_expected_still_wins(self) -> None: + actual = {"Web_Search": 1, "webSearch1": 1} + expected = {"webSearch1": (">=", 1)} + score, _ = tool_calls_count_score(actual, expected) + assert score == 1.0 diff --git a/packages/uipath/tests/evaluators/test_evaluator_methods.py b/packages/uipath/tests/evaluators/test_evaluator_methods.py index 22cfc980e..98efb94b1 100644 --- a/packages/uipath/tests/evaluators/test_evaluator_methods.py +++ b/packages/uipath/tests/evaluators/test_evaluator_methods.py @@ -58,26 +58,26 @@ ) from uipath.eval.models import NumericEvaluationResult from uipath.eval.models.models import ( - AgentExecution, ToolCall, ToolOutput, UiPathEvaluationError, + WorkloadExecution, ) @pytest.fixture -def sample_agent_execution() -> AgentExecution: - """Create a sample AgentExecution for testing.""" - return AgentExecution( +def sample_agent_execution() -> WorkloadExecution: + """Create a sample WorkloadExecution for testing.""" + return WorkloadExecution( agent_input={"input": "Test input"}, - agent_output={"output": "Test output"}, - agent_trace=[], # Empty trace for basic tests + workload_output={"output": "Test output"}, + workload_trace=[], # Empty trace for basic tests ) @pytest.fixture -def sample_agent_execution_with_trace() -> AgentExecution: - """Create a sample AgentExecution with tool call trace.""" +def sample_agent_execution_with_trace() -> WorkloadExecution: + """Create a sample WorkloadExecution with tool call trace.""" # Mock spans that represent tool calls - simplified for testing mock_spans = [ ReadableSpan( @@ -122,12 +122,12 @@ def sample_agent_execution_with_trace() -> AgentExecution: ), ] - return AgentExecution( + return WorkloadExecution( agent_input={"input": "Test input with tools"}, - agent_output={ + workload_output={ "output": "Test output with tools", }, - agent_trace=mock_spans, + workload_trace=mock_spans, ) @@ -136,7 +136,7 @@ class TestExactMatchEvaluator: @pytest.mark.asyncio async def test_exact_match_string_success( - self, sample_agent_execution: AgentExecution + self, sample_agent_execution: WorkloadExecution ) -> None: """Test exact match with matching strings.""" config = { @@ -156,7 +156,7 @@ async def test_exact_match_string_success( @pytest.mark.asyncio async def test_exact_match_string_failure( - self, sample_agent_execution: AgentExecution + self, sample_agent_execution: WorkloadExecution ) -> None: """Test exact match with non-matching strings.""" config = { @@ -178,7 +178,7 @@ async def test_exact_match_string_failure( @pytest.mark.asyncio async def test_exact_match_negated( - self, sample_agent_execution: AgentExecution + self, sample_agent_execution: WorkloadExecution ) -> None: """Test exact match with negated criteria.""" config = { @@ -215,10 +215,10 @@ async def test_exact_match_numeric_normalization( self, actual_output: Any, expected_output: Any, expected_score: float ) -> None: """Test that int and float scalar values are normalized before comparison.""" - execution = AgentExecution( + execution = WorkloadExecution( agent_input={}, - agent_output={"value": actual_output}, - agent_trace=[], + workload_output={"value": actual_output}, + workload_trace=[], ) config = {"name": "ExactMatchNumericTest", "target_output_key": "value"} evaluator = ExactMatchEvaluator.model_validate( @@ -260,10 +260,10 @@ async def test_exact_match_recursive_normalization( expected_score: float, ) -> None: """Test that int/float normalization works recursively for dicts, lists, and nested structures.""" - execution = AgentExecution( + execution = WorkloadExecution( agent_input={}, - agent_output=actual_output, - agent_trace=[], + workload_output=actual_output, + workload_trace=[], ) config = {"name": "ExactMatchRecursiveTest", "target_output_key": target_key} evaluator = ExactMatchEvaluator.model_validate( @@ -278,7 +278,7 @@ async def test_exact_match_recursive_normalization( @pytest.mark.asyncio async def test_exact_match_validate_and_evaluate_criteria( - self, sample_agent_execution: AgentExecution + self, sample_agent_execution: WorkloadExecution ) -> None: """Test exact match using validate_and_evaluate_criteria.""" config = { @@ -311,15 +311,15 @@ async def test_exact_match_line_by_line_all_match(self) -> None: ) # Multi-line output - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"input": "Test input"}, - agent_output="line1\nline2\nline3", - agent_trace=[], + workload_output="line1\nline2\nline3", + workload_trace=[], ) criteria = OutputEvaluationCriteria(expected_output="line1\nline2\nline3") # pyright: ignore[reportCallIssue] result = await evaluator.validate_and_evaluate_criteria( - agent_execution, criteria + workload_execution, criteria ) assert isinstance(result, NumericEvaluationResult) @@ -346,15 +346,15 @@ async def test_exact_match_line_by_line_partial_match(self) -> None: ) # Multi-line output with 2 out of 3 lines matching - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"input": "Test input"}, - agent_output="line1\nwrong\nline3", - agent_trace=[], + workload_output="line1\nwrong\nline3", + workload_trace=[], ) criteria = OutputEvaluationCriteria(expected_output="line1\nline2\nline3") # pyright: ignore[reportCallIssue] result = await evaluator.validate_and_evaluate_criteria( - agent_execution, criteria + workload_execution, criteria ) assert isinstance(result, NumericEvaluationResult) @@ -384,15 +384,15 @@ async def test_exact_match_line_by_line_custom_delimiter(self) -> None: ) # Pipe-delimited output - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"input": "Test input"}, - agent_output="part1|part2|part3", - agent_trace=[], + workload_output="part1|part2|part3", + workload_trace=[], ) criteria = OutputEvaluationCriteria(expected_output="part1|part2|part3") # pyright: ignore[reportCallIssue] result = await evaluator.validate_and_evaluate_criteria( - agent_execution, criteria + workload_execution, criteria ) assert isinstance(result, NumericEvaluationResult) @@ -415,15 +415,15 @@ async def test_exact_match_line_by_line_has_individual_results(self) -> None: ) # Multi-line output with 2 out of 3 lines matching - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"input": "Test input"}, - agent_output="line1\nwrong\nline3", - agent_trace=[], + workload_output="line1\nwrong\nline3", + workload_trace=[], ) criteria = OutputEvaluationCriteria(expected_output="line1\nline2\nline3") # pyright: ignore[reportCallIssue] result = await evaluator.validate_and_evaluate_criteria( - agent_execution, criteria + workload_execution, criteria ) # Check that the result has the _line_by_line_results attribute @@ -447,12 +447,454 @@ async def test_exact_match_line_by_line_has_individual_results(self) -> None: assert line3_result.score == 1.0 +class TestListTargetOutputKey: + """Test target_output_key as a list of keys.""" + + @pytest.mark.asyncio + async def test_exact_match_list_keys_all_match(self) -> None: + """All listed keys match → score 1.0.""" + execution = WorkloadExecution( + agent_input={}, + workload_output={"status": "ok", "total": 42, "extra": "ignored"}, + workload_trace=[], + ) + config = { + "name": "ExactMatchListKeys", + "target_output_key": ["status", "total"], + } + evaluator = ExactMatchEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + criteria = OutputEvaluationCriteria( + expected_output={"status": "ok", "total": 42} # pyright: ignore[reportCallIssue] + ) + result = await evaluator.evaluate(execution, criteria) + assert isinstance(result, NumericEvaluationResult) + assert result.score == 1.0 + + @pytest.mark.asyncio + async def test_exact_match_list_keys_value_mismatch(self) -> None: + """One key's value differs → score 0.0.""" + execution = WorkloadExecution( + agent_input={}, + workload_output={"status": "ok", "total": 99}, + workload_trace=[], + ) + config = { + "name": "ExactMatchListKeys", + "target_output_key": ["status", "total"], + } + evaluator = ExactMatchEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + criteria = OutputEvaluationCriteria( + expected_output={"status": "ok", "total": 42} # pyright: ignore[reportCallIssue] + ) + result = await evaluator.evaluate(execution, criteria) + assert isinstance(result, NumericEvaluationResult) + assert result.score == 0.0 + + @pytest.mark.asyncio + async def test_exact_match_list_keys_dot_notation(self) -> None: + """Nested dot-notation paths inside a list of keys.""" + execution = WorkloadExecution( + agent_input={}, + workload_output={"order": {"status": "shipped"}, "qty": 3}, + workload_trace=[], + ) + config = { + "name": "ExactMatchListDotKeys", + "target_output_key": ["order.status", "qty"], + } + evaluator = ExactMatchEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + criteria = OutputEvaluationCriteria( + expected_output={"order": {"status": "shipped"}, "qty": 3} # pyright: ignore[reportCallIssue] + ) + result = await evaluator.evaluate(execution, criteria) + assert isinstance(result, NumericEvaluationResult) + assert result.score == 1.0 + + @pytest.mark.asyncio + async def test_list_keys_missing_key_in_actual_raises(self) -> None: + """Missing key in actual output returns an ErrorEvaluationResult.""" + from uipath.eval.models.models import ErrorEvaluationResult + + execution = WorkloadExecution( + agent_input={}, + workload_output={"status": "ok"}, # 'total' is missing + workload_trace=[], + ) + config = { + "name": "ExactMatchListKeys", + "target_output_key": ["status", "total"], + } + evaluator = ExactMatchEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + criteria = OutputEvaluationCriteria( + expected_output={"status": "ok", "total": 42} # pyright: ignore[reportCallIssue] + ) + result = await evaluator.evaluate(execution, criteria) + assert isinstance(result, ErrorEvaluationResult) + assert result.score == 0.0 + + @pytest.mark.asyncio + async def test_list_keys_missing_key_in_expected_raises(self) -> None: + """Missing key in expected output returns an ErrorEvaluationResult.""" + from uipath.eval.models.models import ErrorEvaluationResult + + execution = WorkloadExecution( + agent_input={}, + workload_output={"status": "ok", "total": 42}, + workload_trace=[], + ) + config = { + "name": "ExactMatchListKeys", + "target_output_key": ["status", "total"], + } + evaluator = ExactMatchEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + criteria = OutputEvaluationCriteria( + expected_output={ + "status": "ok" + } # 'total' missing # pyright: ignore[reportCallIssue] + ) + result = await evaluator.evaluate(execution, criteria) + assert isinstance(result, ErrorEvaluationResult) + assert result.score == 0.0 + + @pytest.mark.asyncio + async def test_list_keys_expected_as_json_string(self) -> None: + """Expected output as a JSON string is parsed when key is a list.""" + import json + + execution = WorkloadExecution( + agent_input={}, + workload_output={"status": "ok", "total": 5}, + workload_trace=[], + ) + config = { + "name": "ExactMatchListKeys", + "target_output_key": ["status", "total"], + } + evaluator = ExactMatchEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + criteria = OutputEvaluationCriteria( + expected_output=json.dumps({"status": "ok", "total": 5}) # pyright: ignore[reportCallIssue] + ) + result = await evaluator.evaluate(execution, criteria) + assert isinstance(result, NumericEvaluationResult) + assert result.score == 1.0 + + @pytest.mark.asyncio + async def test_list_keys_invalid_json_string_expected_raises(self) -> None: + """Invalid JSON string for expected output returns an ErrorEvaluationResult.""" + from uipath.eval.models.models import ErrorEvaluationResult + + execution = WorkloadExecution( + agent_input={}, + workload_output={"status": "ok"}, + workload_trace=[], + ) + config = {"name": "ExactMatchListKeys", "target_output_key": ["status"]} + evaluator = ExactMatchEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + criteria = OutputEvaluationCriteria( + expected_output="not valid json" # pyright: ignore[reportCallIssue] + ) + result = await evaluator.evaluate(execution, criteria) + assert isinstance(result, ErrorEvaluationResult) + assert result.score == 0.0 + + @pytest.mark.asyncio + async def test_list_keys_disables_line_by_line(self) -> None: + """line_by_line_evaluator=True is ignored when target_output_key is a list.""" + execution = WorkloadExecution( + agent_input={}, + workload_output={"a": "x", "b": "y"}, + workload_trace=[], + ) + config = { + "name": "ExactMatchListLbl", + "target_output_key": ["a", "b"], + "line_by_line_evaluator": True, + } + evaluator = ExactMatchEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + raw_criteria = {"expected_output": {"a": "x", "b": "y"}} + # Should not raise or split into lines; returns a plain NumericEvaluationResult + result = await evaluator.validate_and_evaluate_criteria(execution, raw_criteria) + assert isinstance(result, NumericEvaluationResult) + assert result.score == 1.0 + # No line-by-line details attached + assert ( + not hasattr(result, "_line_by_line_results") + or result._line_by_line_results is None + ) + + @pytest.mark.asyncio + async def test_json_similarity_list_keys_perfect_match(self) -> None: + """JsonSimilarityEvaluator with list keys scores 1.0 on exact match.""" + from uipath.eval.evaluators.json_similarity_evaluator import ( + JsonSimilarityEvaluator, + ) + + execution = WorkloadExecution( + agent_input={}, + workload_output={"name": "Alice", "score": 100, "extra": "ignored"}, + workload_trace=[], + ) + config = { + "name": "JsonSimListKeys", + "target_output_key": ["name", "score"], + } + evaluator = JsonSimilarityEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + criteria = OutputEvaluationCriteria( + expected_output={"name": "Alice", "score": 100} # pyright: ignore[reportCallIssue] + ) + result = await evaluator.evaluate(execution, criteria) + assert isinstance(result, NumericEvaluationResult) + assert result.score == 1.0 + + @pytest.mark.asyncio + async def test_list_keys_non_dict_actual_raises(self) -> None: + """Non-dict workload_output with list key returns ErrorEvaluationResult.""" + from uipath.eval.models.models import ErrorEvaluationResult + + execution = WorkloadExecution( + agent_input={}, + workload_output="just a string", # pyright: ignore[reportArgumentType] + workload_trace=[], + ) + config = {"name": "ExactMatchListKeys", "target_output_key": ["status"]} + evaluator = ExactMatchEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + criteria = OutputEvaluationCriteria( + expected_output={"status": "ok"} # pyright: ignore[reportCallIssue] + ) + result = await evaluator.evaluate(execution, criteria) + assert isinstance(result, ErrorEvaluationResult) + assert result.score == 0.0 + + @pytest.mark.asyncio + async def test_list_keys_non_dict_json_expected_raises(self) -> None: + """Valid JSON that parses to a non-dict returns ErrorEvaluationResult.""" + import json + + from uipath.eval.models.models import ErrorEvaluationResult + + execution = WorkloadExecution( + agent_input={}, + workload_output={"status": "ok"}, + workload_trace=[], + ) + config = {"name": "ExactMatchListKeys", "target_output_key": ["status"]} + evaluator = ExactMatchEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + # Valid JSON but not an object — triggers the isinstance(expected_output, dict) guard + criteria = OutputEvaluationCriteria( + expected_output=json.dumps("just a string") # pyright: ignore[reportCallIssue] + ) + result = await evaluator.evaluate(execution, criteria) + assert isinstance(result, ErrorEvaluationResult) + assert result.score == 0.0 + + @pytest.mark.asyncio + async def test_list_keys_attachment_uri_downloaded( + self, mocker: MockerFixture + ) -> None: + """Attachment URIs within list-key actual output values are downloaded.""" + att_uri = ( + "urn:uipath:cas:file:orchestrator:00000000-0000-0000-0000-000000000001" + ) + mocker.patch( + "uipath.eval.evaluators.output_evaluator.download_attachment_as_string", + return_value="downloaded_content", + ) + execution = WorkloadExecution( + agent_input={}, + workload_output={"file": att_uri, "status": "ok"}, + workload_trace=[], + ) + config = {"name": "ExactMatchListKeys", "target_output_key": ["file", "status"]} + evaluator = ExactMatchEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + criteria = OutputEvaluationCriteria( + expected_output={"file": "downloaded_content", "status": "ok"} # pyright: ignore[reportCallIssue] + ) + result = await evaluator.evaluate(execution, criteria) + assert isinstance(result, NumericEvaluationResult) + assert result.score == 1.0 + + @pytest.mark.asyncio + async def test_scalar_key_attachment_uri_downloaded( + self, mocker: MockerFixture + ) -> None: + """Attachment URI in scalar-key actual output is downloaded.""" + att_uri = ( + "urn:uipath:cas:file:orchestrator:00000000-0000-0000-0000-000000000002" + ) + mocker.patch( + "uipath.eval.evaluators.output_evaluator.download_attachment_as_string", + return_value="file_content", + ) + execution = WorkloadExecution( + agent_input={}, + workload_output={"report": att_uri}, + workload_trace=[], + ) + config = {"name": "ExactMatchScalarAtt", "target_output_key": "report"} + evaluator = ExactMatchEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + criteria = OutputEvaluationCriteria( + expected_output={"report": "file_content"} # pyright: ignore[reportCallIssue] + ) + result = await evaluator.evaluate(execution, criteria) + assert isinstance(result, NumericEvaluationResult) + assert result.score == 1.0 + + @pytest.mark.asyncio + async def test_scalar_key_missing_in_actual_raises(self) -> None: + """Missing scalar target_output_key in actual output returns ErrorEvaluationResult.""" + from uipath.eval.models.models import ErrorEvaluationResult + + execution = WorkloadExecution( + agent_input={}, + workload_output={"other": "value"}, + workload_trace=[], + ) + config = {"name": "ExactMatchMissingActual", "target_output_key": "missing_key"} + evaluator = ExactMatchEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + criteria = OutputEvaluationCriteria( + expected_output={"missing_key": "val"} # pyright: ignore[reportCallIssue] + ) + result = await evaluator.evaluate(execution, criteria) + assert isinstance(result, ErrorEvaluationResult) + assert result.score == 0.0 + + @pytest.mark.asyncio + async def test_scalar_key_missing_in_expected_raises(self) -> None: + """Missing scalar target_output_key in expected output returns ErrorEvaluationResult.""" + from uipath.eval.models.models import ErrorEvaluationResult + + execution = WorkloadExecution( + agent_input={}, + workload_output={"status": "ok"}, + workload_trace=[], + ) + config = {"name": "ExactMatchMissingExpected", "target_output_key": "status"} + evaluator = ExactMatchEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + # expected output dict doesn't contain the target key + criteria = OutputEvaluationCriteria( + expected_output={"other_key": "ok"} # pyright: ignore[reportCallIssue] + ) + result = await evaluator.evaluate(execution, criteria) + assert isinstance(result, ErrorEvaluationResult) + assert result.score == 0.0 + + @pytest.mark.asyncio + async def test_scalar_key_invalid_json_expected_raises(self) -> None: + """Invalid JSON string for expected output with scalar key returns ErrorEvaluationResult.""" + from uipath.eval.models.models import ErrorEvaluationResult + + execution = WorkloadExecution( + agent_input={}, + workload_output={"status": "ok"}, + workload_trace=[], + ) + config = {"name": "ExactMatchInvalidJson", "target_output_key": "status"} + evaluator = ExactMatchEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + criteria = OutputEvaluationCriteria( + expected_output="not valid json" # pyright: ignore[reportCallIssue] + ) + result = await evaluator.evaluate(execution, criteria) + assert isinstance(result, ErrorEvaluationResult) + assert result.score == 0.0 + + @pytest.mark.asyncio + async def test_validate_and_evaluate_criteria_none_raises(self) -> None: + """None criteria with no default configured raises UiPathEvaluationError.""" + execution = WorkloadExecution( + agent_input={}, + workload_output={"status": "ok"}, + workload_trace=[], + ) + config = {"name": "ExactMatchNoCriteria"} + evaluator = ExactMatchEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + with pytest.raises(UiPathEvaluationError) as exc_info: + await evaluator.validate_and_evaluate_criteria(execution, None) + assert "MISSING_EVALUATION_CRITERIA" in exc_info.value.error_info.code + + def test_base_output_evaluator_get_full_expected_output_raises(self) -> None: + """BaseOutputEvaluator._get_full_expected_output raises NOT_IMPLEMENTED.""" + from uipath.eval.evaluators.base_evaluator import BaseEvaluatorJustification + from uipath.eval.evaluators.output_evaluator import ( + BaseOutputEvaluator, + OutputEvaluationCriteria, + OutputEvaluatorConfig, + ) + from uipath.eval.models import EvaluationResult + + class _MinimalEvaluator( + BaseOutputEvaluator[ + OutputEvaluationCriteria, + OutputEvaluatorConfig[OutputEvaluationCriteria], + BaseEvaluatorJustification, + ] + ): + @classmethod + def get_evaluator_id(cls) -> str: + return "uipath-minimal-test" + + async def evaluate( + self, workload_execution: Any, evaluation_criteria: Any + ) -> EvaluationResult: + return None # type: ignore[return-value] + + def validate_evaluation_criteria( + self, raw: Any + ) -> OutputEvaluationCriteria: + return OutputEvaluationCriteria.model_validate(raw) + + evaluator = _MinimalEvaluator.model_validate( + { + "evaluatorConfig": {"name": "minimal"}, + "id": str(uuid.uuid4()), + } + ) + with pytest.raises(UiPathEvaluationError) as exc_info: + evaluator._get_full_expected_output( # pyright: ignore[reportArgumentType] + OutputEvaluationCriteria(expected_output={}) # pyright: ignore[reportCallIssue] + ) + assert "NOT_IMPLEMENTED" in exc_info.value.error_info.code + + class TestContainsEvaluator: """Test ContainsEvaluator.evaluate() method.""" @pytest.mark.asyncio @pytest.mark.parametrize( - "agent_output, search_text, target_key, case_sensitive, negated, expected_score", + "workload_output, search_text, target_key, case_sensitive, negated, expected_score", [ # Basic match ("Test output", "Test output", "*", False, False, 1.0), @@ -476,24 +918,24 @@ class TestContainsEvaluator: ) async def test_contains_evaluator( self, - agent_output: Any, + workload_output: Any, search_text: str, target_key: str, case_sensitive: bool, negated: bool, expected_score: float, - sample_agent_execution: AgentExecution, + sample_agent_execution: WorkloadExecution, ) -> None: """Test ContainsEvaluator across match, no-match, case sensitivity, and negation cases.""" if target_key == "output": execution = ( - sample_agent_execution # has agent_output={"output": "Test output"} + sample_agent_execution # has workload_output={"output": "Test output"} ) else: - execution = AgentExecution( + execution = WorkloadExecution( agent_input={}, - agent_output=agent_output, - agent_trace=[], + workload_output=workload_output, + workload_trace=[], ) config = { "name": "ContainsTest", @@ -512,7 +954,7 @@ async def test_contains_evaluator( @pytest.mark.asyncio async def test_contains_evaluator_validate_and_evaluate_criteria( - self, sample_agent_execution: AgentExecution + self, sample_agent_execution: WorkloadExecution ) -> None: """Test contains evaluator with validate_and_evaluate_criteria.""" config = { @@ -536,10 +978,10 @@ class TestJsonSimilarityEvaluator: @pytest.mark.asyncio async def test_json_similarity_identical(self) -> None: """Test JSON similarity with identical structures.""" - execution = AgentExecution( + execution = WorkloadExecution( agent_input={"input": "Test"}, - agent_output={"name": "John", "age": 30, "city": "NYC"}, - agent_trace=[], + workload_output={"name": "John", "age": 30, "city": "NYC"}, + workload_trace=[], ) config = { "name": "JsonSimilarityTest", @@ -559,10 +1001,10 @@ async def test_json_similarity_identical(self) -> None: @pytest.mark.asyncio async def test_json_similarity_partial_match(self) -> None: """Test JSON similarity with partial matches.""" - execution = AgentExecution( + execution = WorkloadExecution( agent_input={"input": "Test"}, - agent_output={"name": "John", "age": 30, "city": "LA"}, - agent_trace=[], + workload_output={"name": "John", "age": 30, "city": "LA"}, + workload_trace=[], ) config = { "name": "JsonSimilarityTest", @@ -602,10 +1044,10 @@ async def test_json_similarity_numeric_normalization( expected_score: float, ) -> None: """Test that int/float normalization is applied before JSON similarity comparison.""" - execution = AgentExecution( + execution = WorkloadExecution( agent_input={}, - agent_output=actual_output, - agent_trace=[], + workload_output=actual_output, + workload_trace=[], ) config = {"name": "JsonSimilarityTest"} evaluator = JsonSimilarityEvaluator.model_validate( @@ -621,10 +1063,10 @@ async def test_json_similarity_numeric_normalization( @pytest.mark.asyncio async def test_json_similarity_validate_and_evaluate_criteria(self) -> None: """Test JSON similarity using validate_and_evaluate_criteria.""" - execution = AgentExecution( + execution = WorkloadExecution( agent_input={"input": "Test"}, - agent_output={"name": "John", "age": 30, "city": "NYC"}, - agent_trace=[], + workload_output={"name": "John", "age": 30, "city": "NYC"}, + workload_trace=[], ) config = { "name": "JsonSimilarityTest", @@ -645,7 +1087,7 @@ class TestToolCallOrderEvaluator: @pytest.mark.asyncio async def test_tool_call_order_perfect_match( - self, sample_agent_execution_with_trace: AgentExecution + self, sample_agent_execution_with_trace: WorkloadExecution ) -> None: """Test tool call order with perfect order match.""" @@ -668,7 +1110,7 @@ async def test_tool_call_order_perfect_match( @pytest.mark.asyncio async def test_tool_call_order_no_perfect_match( - self, sample_agent_execution_with_trace: AgentExecution + self, sample_agent_execution_with_trace: WorkloadExecution ) -> None: """Test tool call order with perfect order match.""" @@ -691,7 +1133,7 @@ async def test_tool_call_order_no_perfect_match( @pytest.mark.asyncio async def test_tool_call_order_lcs_match( - self, sample_agent_execution_with_trace: AgentExecution + self, sample_agent_execution_with_trace: WorkloadExecution ) -> None: """Test tool call order with lcs order match.""" @@ -713,7 +1155,7 @@ async def test_tool_call_order_lcs_match( @pytest.mark.asyncio async def test_tool_call_order_validate_and_evaluate_criteria( - self, sample_agent_execution_with_trace: AgentExecution + self, sample_agent_execution_with_trace: WorkloadExecution ) -> None: """Test tool call order using validate_and_evaluate_criteria.""" config = { @@ -738,7 +1180,7 @@ class TestToolCallCountEvaluator: @pytest.mark.asyncio async def test_tool_call_count_exact_match( - self, sample_agent_execution_with_trace: AgentExecution + self, sample_agent_execution_with_trace: WorkloadExecution ) -> None: """Test tool call count with exact count match.""" config = { @@ -759,7 +1201,7 @@ async def test_tool_call_count_exact_match( @pytest.mark.asyncio async def test_tool_call_count_with_gt( - self, sample_agent_execution_with_trace: AgentExecution + self, sample_agent_execution_with_trace: WorkloadExecution ) -> None: """Test tool call count with strict count match.""" config = { @@ -780,7 +1222,7 @@ async def test_tool_call_count_with_gt( @pytest.mark.asyncio async def test_tool_call_count_no_exact_match( - self, sample_agent_execution_with_trace: AgentExecution + self, sample_agent_execution_with_trace: WorkloadExecution ) -> None: """Test tool call count with no exact count match.""" config = { @@ -801,7 +1243,7 @@ async def test_tool_call_count_no_exact_match( @pytest.mark.asyncio async def test_tool_call_count_partial_match( - self, sample_agent_execution_with_trace: AgentExecution + self, sample_agent_execution_with_trace: WorkloadExecution ) -> None: """Test tool call count with partial count match.""" config = { @@ -822,7 +1264,7 @@ async def test_tool_call_count_partial_match( @pytest.mark.asyncio async def test_tool_call_count_validate_and_evaluate_criteria( - self, sample_agent_execution_with_trace: AgentExecution + self, sample_agent_execution_with_trace: WorkloadExecution ) -> None: """Test tool call count using validate_and_evaluate_criteria.""" config = { @@ -847,7 +1289,7 @@ class TestToolCallArgsEvaluator: @pytest.mark.asyncio async def test_tool_call_args_perfect_match( - self, sample_agent_execution_with_trace: AgentExecution + self, sample_agent_execution_with_trace: WorkloadExecution ) -> None: """Test tool call args with perfect match.""" config = { @@ -873,7 +1315,7 @@ async def test_tool_call_args_perfect_match( @pytest.mark.asyncio async def test_tool_call_args_partial_match( - self, sample_agent_execution_with_trace: AgentExecution + self, sample_agent_execution_with_trace: WorkloadExecution ) -> None: """Test tool call args with partial match.""" config = { @@ -899,7 +1341,7 @@ async def test_tool_call_args_partial_match( @pytest.mark.asyncio async def test_tool_call_args_validate_and_evaluate_criteria( - self, sample_agent_execution_with_trace: AgentExecution + self, sample_agent_execution_with_trace: WorkloadExecution ) -> None: """Test tool call args using validate_and_evaluate_criteria.""" config = { @@ -931,7 +1373,7 @@ class TestToolCallOutputEvaluator: @pytest.mark.asyncio async def test_tool_call_output_perfect_match( - self, sample_agent_execution_with_trace: AgentExecution + self, sample_agent_execution_with_trace: WorkloadExecution ) -> None: """Test tool call output with perfect output match.""" config = { @@ -957,7 +1399,7 @@ async def test_tool_call_output_perfect_match( @pytest.mark.asyncio async def test_tool_call_output_partial_match( - self, sample_agent_execution_with_trace: AgentExecution + self, sample_agent_execution_with_trace: WorkloadExecution ) -> None: """Test tool call output with partial output match.""" config = { @@ -983,7 +1425,7 @@ async def test_tool_call_output_partial_match( @pytest.mark.asyncio async def test_tool_call_output_no_match_strict( - self, sample_agent_execution_with_trace: AgentExecution + self, sample_agent_execution_with_trace: WorkloadExecution ) -> None: """Test tool call output with no match in strict mode.""" config = { @@ -1009,7 +1451,7 @@ async def test_tool_call_output_no_match_strict( @pytest.mark.asyncio async def test_tool_call_output_partial_match_non_strict( - self, sample_agent_execution_with_trace: AgentExecution + self, sample_agent_execution_with_trace: WorkloadExecution ) -> None: """Test tool call output with partial match in non-strict mode.""" config = { @@ -1033,7 +1475,7 @@ async def test_tool_call_output_partial_match_non_strict( @pytest.mark.asyncio async def test_tool_call_output_empty_criteria( - self, sample_agent_execution_with_trace: AgentExecution + self, sample_agent_execution_with_trace: WorkloadExecution ) -> None: """Test tool call output with empty criteria.""" config = { @@ -1052,7 +1494,7 @@ async def test_tool_call_output_empty_criteria( @pytest.mark.asyncio async def test_tool_call_output_validate_and_evaluate_criteria( - self, sample_agent_execution_with_trace: AgentExecution + self, sample_agent_execution_with_trace: WorkloadExecution ) -> None: """Test tool call output using validate_and_evaluate_criteria.""" config = { @@ -1084,7 +1526,7 @@ class TestLlmAsAJudgeEvaluator: @pytest.mark.asyncio async def test_llm_judge_basic_evaluation( - self, sample_agent_execution: AgentExecution, mocker: MockerFixture + self, sample_agent_execution: WorkloadExecution, mocker: MockerFixture ) -> None: """Test LLM as judge basic evaluation functionality with function calling.""" mock_tool_call = mocker.MagicMock() @@ -1134,7 +1576,7 @@ async def mock_chat_completions(*args: Any, **kwargs: Any) -> Any: @pytest.mark.asyncio async def test_llm_judge_basic_evaluation_with_llm_service( - self, sample_agent_execution: AgentExecution, mocker: MockerFixture + self, sample_agent_execution: WorkloadExecution, mocker: MockerFixture ) -> None: """Test LLM judge basic evaluation functionality with a custom LLM service and function calling.""" # Mock tool call for function calling approach @@ -1181,7 +1623,7 @@ async def mock_chat_completions(*args: Any, **kwargs: Any) -> Any: @pytest.mark.asyncio async def test_llm_judge_validate_and_evaluate_criteria( - self, sample_agent_execution: AgentExecution, mocker: MockerFixture + self, sample_agent_execution: WorkloadExecution, mocker: MockerFixture ) -> None: """Test LLM judge using validate_and_evaluate_criteria with function calling.""" mock_tool_call = mocker.MagicMock() @@ -1236,7 +1678,7 @@ class TestLlmJudgeTrajectoryEvaluator: @pytest.mark.asyncio async def test_llm_trajectory_basic_evaluation( - self, sample_agent_execution: AgentExecution, mocker: MockerFixture + self, sample_agent_execution: WorkloadExecution, mocker: MockerFixture ) -> None: """Test LLM trajectory judge basic evaluation functionality with function calling.""" mock_tool_call = mocker.MagicMock() @@ -1288,7 +1730,7 @@ async def mock_chat_completions(*args: Any, **kwargs: Any) -> Any: @pytest.mark.asyncio async def test_llm_trajectory_validate_and_evaluate_criteria( - self, sample_agent_execution: AgentExecution, mocker: MockerFixture + self, sample_agent_execution: WorkloadExecution, mocker: MockerFixture ) -> None: """Test LLM trajectory judge using validate_and_evaluate_criteria with function calling.""" mock_tool_call = mocker.MagicMock() @@ -1383,7 +1825,7 @@ class TestEvaluationResultTypes: @pytest.mark.asyncio async def test_evaluators_return_results_with_scores( - self, sample_agent_execution: AgentExecution + self, sample_agent_execution: WorkloadExecution ) -> None: """Test that evaluators return results with scores.""" config = { @@ -1405,7 +1847,7 @@ class TestJustificationHandling: @pytest.mark.asyncio async def test_exact_match_evaluator_justification( - self, sample_agent_execution: AgentExecution + self, sample_agent_execution: WorkloadExecution ) -> None: """Test that ExactMatchEvaluator provides BaseEvaluatorJustification.""" @@ -1430,10 +1872,10 @@ async def test_exact_match_evaluator_justification( async def test_json_similarity_evaluator_justification(self) -> None: """Test that JsonSimilarityEvaluator provides JsonSimilarityJustification.""" - execution = AgentExecution( + execution = WorkloadExecution( agent_input={"input": "Test"}, - agent_output={"name": "John", "age": 30, "city": "NYC"}, - agent_trace=[], + workload_output={"name": "John", "age": 30, "city": "NYC"}, + workload_trace=[], ) config = { "name": "JsonSimilarityTest", @@ -1455,7 +1897,7 @@ async def test_json_similarity_evaluator_justification(self) -> None: @pytest.mark.asyncio async def test_tool_call_order_evaluator_justification( - self, sample_agent_execution_with_trace: AgentExecution + self, sample_agent_execution_with_trace: WorkloadExecution ) -> None: """Test that ToolCallOrderEvaluator provides structured justification.""" @@ -1478,7 +1920,7 @@ async def test_tool_call_order_evaluator_justification( @pytest.mark.asyncio async def test_tool_call_count_evaluator_justification( - self, sample_agent_execution_with_trace: AgentExecution + self, sample_agent_execution_with_trace: WorkloadExecution ) -> None: """Test that ToolCallCountEvaluator provides structured justification.""" @@ -1501,7 +1943,7 @@ async def test_tool_call_count_evaluator_justification( @pytest.mark.asyncio async def test_tool_call_args_evaluator_justification( - self, sample_agent_execution_with_trace: AgentExecution + self, sample_agent_execution_with_trace: WorkloadExecution ) -> None: """Test that ToolCallArgsEvaluator provides structured justification.""" @@ -1529,7 +1971,7 @@ async def test_tool_call_args_evaluator_justification( @pytest.mark.asyncio async def test_tool_call_output_evaluator_justification( - self, sample_agent_execution_with_trace: AgentExecution + self, sample_agent_execution_with_trace: WorkloadExecution ) -> None: """Test that ToolCallOutputEvaluator handles justification correctly.""" config = { @@ -1561,7 +2003,7 @@ async def test_tool_call_output_evaluator_justification( @pytest.mark.asyncio async def test_llm_judge_output_evaluator_justification( - self, sample_agent_execution: AgentExecution, mocker: MockerFixture + self, sample_agent_execution: WorkloadExecution, mocker: MockerFixture ) -> None: """Test that LLMJudgeOutputEvaluator handles str justification correctly with function calling.""" mock_tool_call = mocker.MagicMock() @@ -1617,7 +2059,7 @@ async def mock_chat_completions(*args: Any, **kwargs: Any) -> Any: @pytest.mark.asyncio async def test_llm_judge_trajectory_evaluator_justification( - self, sample_agent_execution: AgentExecution, mocker: MockerFixture + self, sample_agent_execution: WorkloadExecution, mocker: MockerFixture ) -> None: """Test that LLMJudgeTrajectoryEvaluator handles str justification correctly.""" mock_tool_call = mocker.MagicMock() @@ -1763,7 +2205,7 @@ def test_justification_type_extraction_all_evaluators(self) -> None: @pytest.mark.asyncio async def test_llm_judge_omits_max_tokens_when_none( - self, sample_agent_execution: AgentExecution, mocker: MockerFixture + self, sample_agent_execution: WorkloadExecution, mocker: MockerFixture ) -> None: """Test that max_tokens is omitted from API request when None (fixes 400 error).""" mock_tool_call = mocker.MagicMock() @@ -1810,7 +2252,7 @@ async def capture_chat_completions(**kwargs: Any) -> Any: ) result = await evaluator.evaluate( - agent_execution=sample_agent_execution, + workload_execution=sample_agent_execution, evaluation_criteria=OutputEvaluationCriteria(expected_output="42"), ) @@ -1850,7 +2292,7 @@ class TestClaude45ModelSupport: async def test_claude_45_evaluator_uses_function_calling( self, model_name: str, - sample_agent_execution: AgentExecution, + sample_agent_execution: WorkloadExecution, mocker: MockerFixture, ) -> None: """Test that Claude 4.5 evaluators use function calling (tools/tool_choice).""" @@ -1923,7 +2365,7 @@ async def capture_chat_completions(**kwargs: Any) -> Any: async def test_claude_45_sets_default_max_tokens( self, model_name: str, - sample_agent_execution: AgentExecution, + sample_agent_execution: WorkloadExecution, mocker: MockerFixture, ) -> None: """Test that Claude 4.5 models get default max_tokens=8000 when not configured.""" @@ -1978,7 +2420,7 @@ async def capture_chat_completions(**kwargs: Any) -> Any: @pytest.mark.asyncio async def test_claude_45_respects_configured_max_tokens( self, - sample_agent_execution: AgentExecution, + sample_agent_execution: WorkloadExecution, mocker: MockerFixture, ) -> None: """Test that explicitly configured max_tokens overrides the Claude 4.5 default.""" @@ -2060,10 +2502,10 @@ async def test_binary_classification_scoring( BinaryClassificationEvaluator, ) - execution = AgentExecution( + execution = WorkloadExecution( agent_input={}, - agent_output={"class": predicted}, - agent_trace=[], + workload_output={"class": predicted}, + workload_trace=[], ) config = { "name": "BinaryClassificationTest", @@ -2115,10 +2557,10 @@ async def test_multiclass_classification_scoring( MulticlassClassificationEvaluator, ) - execution = AgentExecution( + execution = WorkloadExecution( agent_input={}, - agent_output={"class": predicted}, - agent_trace=[], + workload_output={"class": predicted}, + workload_trace=[], ) config = { "name": "MulticlassClassificationTest", @@ -2146,10 +2588,10 @@ async def test_multiclass_classification_invalid_expected_class(self) -> None: ) from uipath.eval.models.models import ErrorEvaluationResult - execution = AgentExecution( + execution = WorkloadExecution( agent_input={}, - agent_output={"class": "cat"}, - agent_trace=[], + workload_output={"class": "cat"}, + workload_trace=[], ) config = { "name": "MulticlassClassificationTest", @@ -2173,10 +2615,10 @@ async def test_multiclass_classification_invalid_predicted_class(self) -> None: ) from uipath.eval.models.models import ErrorEvaluationResult - execution = AgentExecution( + execution = WorkloadExecution( agent_input={}, - agent_output={"class": "fish"}, - agent_trace=[], + workload_output={"class": "fish"}, + workload_trace=[], ) config = { "name": "MulticlassClassificationTest", diff --git a/packages/uipath/tests/evaluators/test_exact_match_aggregators.py b/packages/uipath/tests/evaluators/test_exact_match_aggregators.py new file mode 100644 index 000000000..49ede250e --- /dev/null +++ b/packages/uipath/tests/evaluators/test_exact_match_aggregators.py @@ -0,0 +1,195 @@ +"""ExactMatch aggregator config surface. + +The platform's dataset-evaluator pass (Agents repo) reads ``classes`` and +``aggregators`` off the ExactMatch evaluator config. These tests pin the SDK +side of that wire contract: the discriminated spec union, the config fields, +and the config validator. The aggregation math lives in +``classification_dataset_evaluators.py`` (tested separately) and is consumed +by both `uipath eval` and the platform's python-dataset-eval-worker. +""" + +import uuid +from typing import Any + +import pytest +from pydantic import TypeAdapter, ValidationError + +from uipath.eval.evaluators._aggregator_specs import ( + AggregatorSpec, + ConfusionMatrixAggregatorSpec, + FScoreAggregatorSpec, + PrecisionAggregatorSpec, + RecallAggregatorSpec, +) +from uipath.eval.evaluators.exact_match_evaluator import ExactMatchEvaluator + + +def _evaluator(config: dict[str, Any]) -> ExactMatchEvaluator: + return ExactMatchEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + + +class TestAggregatorSpecUnion: + """Wire shape sent to the platform — {type, averaging, [fValue]}.""" + + def test_discriminates_on_type(self) -> None: + adapter: TypeAdapter[AggregatorSpec] = TypeAdapter(AggregatorSpec) + assert isinstance( + adapter.validate_python({"type": "precision", "averaging": "macro"}), + PrecisionAggregatorSpec, + ) + assert isinstance( + adapter.validate_python({"type": "recall", "averaging": "micro"}), + RecallAggregatorSpec, + ) + fscore = adapter.validate_python( + {"type": "fscore", "averaging": "macro", "fValue": 2.0} + ) + assert isinstance(fscore, FScoreAggregatorSpec) + assert fscore.f_value == 2.0 + assert isinstance( + adapter.validate_python({"type": "confusion_matrix"}), + ConfusionMatrixAggregatorSpec, + ) + + def test_specs_do_not_carry_classes(self) -> None: + # Classes live once on the evaluator config, shared by all aggregators. + dumped = PrecisionAggregatorSpec(averaging="macro").model_dump(by_alias=True) + assert dumped == {"type": "precision", "averaging": "macro"} + + def test_fscore_uses_camelcase_fvalue_on_wire(self) -> None: + dumped = FScoreAggregatorSpec(averaging="macro", f_value=1.5).model_dump( + by_alias=True + ) + assert dumped["fValue"] == 1.5 + assert "f_value" not in dumped + + def test_fscore_fvalue_is_bounded(self) -> None: + # A huge beta overflows beta² to inf → NaN score → unrepresentable JSON. + adapter: TypeAdapter[AggregatorSpec] = TypeAdapter(AggregatorSpec) + for bad in (0, -1, 1e200, float("inf"), float("nan")): + with pytest.raises(ValidationError): + adapter.validate_python( + {"type": "fscore", "averaging": "macro", "fValue": bad} + ) + + +class TestExactMatchAggregatorConfig: + def test_accepts_aggregators_with_classes(self) -> None: + ev = _evaluator( + { + "name": "IntentClassifier", + "classes": ["book", "cancel", "reschedule"], + "aggregators": [ + {"type": "precision", "averaging": "macro"}, + {"type": "recall", "averaging": "macro"}, + {"type": "fscore", "averaging": "macro", "fValue": 1.0}, + ], + } + ) + config = ev.evaluator_config + assert config.classes == ["book", "cancel", "reschedule"] + assert config.aggregators is not None + assert [s.type for s in config.aggregators] == ["precision", "recall", "fscore"] + + def test_rejects_aggregators_without_classes(self) -> None: + # The SDK wraps pydantic's ValidationError in UiPathEvaluationError at + # evaluator construction; match the message rather than the type. + with pytest.raises(Exception, match="classes"): + _evaluator( + { + "name": "IntentClassifier", + "aggregators": [{"type": "precision", "averaging": "macro"}], + } + ) + + def test_rejects_case_duplicate_classes(self) -> None: + # Labels match case-insensitively, so "Yes"/"yes" would collapse onto + # one matrix index and silently skew every metric. + with pytest.raises(Exception, match="unique"): + _evaluator( + { + "name": "IntentClassifier", + "classes": ["Yes", "yes"], + "aggregators": [{"type": "precision", "averaging": "macro"}], + } + ) + + def test_rejects_blank_class_labels(self) -> None: + with pytest.raises(Exception, match="non-blank"): + _evaluator( + { + "name": "IntentClassifier", + "classes": ["yes", " "], + "aggregators": [{"type": "precision", "averaging": "macro"}], + } + ) + + def test_rejects_aggregators_with_case_sensitive(self) -> None: + # Per-datapoint scoring would be case-sensitive while the matrix + # buckets case-insensitively — a 0.0-scored datapoint could land on + # the true-positive diagonal. + with pytest.raises(Exception, match="case_sensitive"): + _evaluator( + { + "name": "IntentClassifier", + "classes": ["yes", "no"], + "caseSensitive": True, + "aggregators": [{"type": "precision", "averaging": "macro"}], + } + ) + + def test_rejects_aggregators_with_negated(self) -> None: + with pytest.raises(Exception, match="negated"): + _evaluator( + { + "name": "IntentClassifier", + "classes": ["yes", "no"], + "negated": True, + "aggregators": [{"type": "precision", "averaging": "macro"}], + } + ) + + def test_rejects_padded_class_labels(self) -> None: + # Padded labels pass a blank check but never match at lookup time — + # every datapoint would silently land in nSkipped. + with pytest.raises(Exception, match="whitespace"): + _evaluator( + { + "name": "IntentClassifier", + "classes": ["yes", "no "], + "aggregators": [{"type": "precision", "averaging": "macro"}], + } + ) + + def test_rejects_aggregators_with_line_by_line(self) -> None: + # Per-line results carry no expected/actual labels — every datapoint + # would land in n_skipped and all metrics would silently read 0. + with pytest.raises(Exception, match="line_by_line"): + _evaluator( + { + "name": "IntentClassifier", + "classes": ["yes", "no"], + "lineByLineEvaluator": True, + "aggregators": [{"type": "precision", "averaging": "macro"}], + } + ) + + def test_plain_config_needs_neither_field(self) -> None: + ev = _evaluator({"name": "PlainMatch"}) + assert ev.evaluator_config.classes is None + assert ev.evaluator_config.aggregators is None + + def test_round_trips_through_dump_and_load(self) -> None: + config = { + "name": "IntentClassifier", + "classes": ["yes", "no"], + "aggregators": [{"type": "fscore", "averaging": "micro", "fValue": 2.0}], + } + ev = _evaluator(config) + dumped = ev.evaluator_config.model_dump(by_alias=True, exclude_none=True) + assert dumped["classes"] == ["yes", "no"] + assert dumped["aggregators"] == [ + {"type": "fscore", "averaging": "micro", "fValue": 2.0} + ] diff --git a/packages/uipath/tests/evaluators/test_legacy_llm_helpers.py b/packages/uipath/tests/evaluators/test_legacy_llm_helpers.py new file mode 100644 index 000000000..c44f8d855 --- /dev/null +++ b/packages/uipath/tests/evaluators/test_legacy_llm_helpers.py @@ -0,0 +1,84 @@ +"""Tests for legacy LLM helper functions (submit_evaluation tool-call parsing).""" + +from types import SimpleNamespace +from typing import Any + +import pytest + +from uipath.eval.evaluators.legacy_llm_helpers import extract_tool_call_response + + +def _make_response(arguments: dict[str, Any]) -> Any: + """Build a minimal chat-completions response carrying a submit_evaluation tool call.""" + tool_call = SimpleNamespace(arguments=arguments) + message = SimpleNamespace(tool_calls=[tool_call]) + choice = SimpleNamespace(message=message) + return SimpleNamespace(choices=[choice]) + + +class TestExtractToolCallResponse: + """Test extract_tool_call_response score validation.""" + + def test_valid_score_passes_through(self) -> None: + response = _make_response({"score": 88, "justification": "ok"}) + + result = extract_tool_call_response(response, "gemini-2.5-flash") + + assert result.score == 88.0 + assert result.justification == "ok" + + def test_out_of_range_score_is_rejected(self) -> None: + # Real payload observed in production: gemini-2.5-flash returned + # score=989898 in its submit_evaluation tool call while the justification + # said the outputs "match perfectly". Unvalidated, this single value blew + # a 64-item run-level average up to 15559.13%. The evaluation must surface + # as an error rather than record a fabricated score. + response = _make_response( + {"score": 989898, "justification": "matches perfectly"} + ) + + with pytest.raises(ValueError, match="Invalid score 989898"): + extract_tool_call_response(response, "gemini-2.5-flash") + + def test_out_of_range_950_is_rejected(self) -> None: + # Second production occurrence from the same eval run: score=950 + # (the model most likely intended 95). + response = _make_response({"score": 950, "justification": "equivalent"}) + + with pytest.raises(ValueError, match="Invalid score 950"): + extract_tool_call_response(response, "gemini-2.5-flash") + + def test_negative_score_is_rejected(self) -> None: + response = _make_response({"score": -5, "justification": "bad"}) + + with pytest.raises(ValueError, match="Invalid score -5"): + extract_tool_call_response(response, "gpt-4o") + + @pytest.mark.parametrize("boundary", [0, 100]) + def test_boundary_scores_accepted(self, boundary: int) -> None: + response = _make_response({"score": boundary, "justification": "j"}) + + result = extract_tool_call_response(response, "m") + + assert result.score == float(boundary) + assert result.justification == "j" + + @pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")]) + def test_non_finite_score_is_rejected(self, value: float) -> None: + response = _make_response({"score": value, "justification": "j"}) + + with pytest.raises(ValueError, match="Invalid score"): + extract_tool_call_response(response, "m") + + def test_missing_score_raises(self) -> None: + response = _make_response({"justification": "j"}) + + with pytest.raises(ValueError, match="Missing 'score'"): + extract_tool_call_response(response, "m") + + @pytest.mark.parametrize("value", ["not-a-number", None, [95]]) + def test_non_numeric_score_is_rejected(self, value: Any) -> None: + response = _make_response({"score": value, "justification": "j"}) + + with pytest.raises(ValueError, match="Non-numeric score"): + extract_tool_call_response(response, "m") diff --git a/packages/uipath/tests/evaluators/test_legacy_target_output_key_paths.py b/packages/uipath/tests/evaluators/test_legacy_target_output_key_paths.py index 01d2235c7..66d5b59cf 100644 --- a/packages/uipath/tests/evaluators/test_legacy_target_output_key_paths.py +++ b/packages/uipath/tests/evaluators/test_legacy_target_output_key_paths.py @@ -14,9 +14,9 @@ ) from uipath.eval.evaluators.base_legacy_evaluator import LegacyEvaluationCriteria from uipath.eval.models.models import ( - AgentExecution, LegacyEvaluatorCategory, LegacyEvaluatorType, + WorkloadExecution, ) NESTED_OUTPUT = { @@ -74,7 +74,9 @@ async def test_nested_dot_path_summary_status(self) -> None: **_make_exact_match_params("summary.status") ) result = await evaluator.evaluate( - AgentExecution(agent_input={}, agent_trace=[], agent_output=NESTED_OUTPUT), + WorkloadExecution( + agent_input={}, workload_trace=[], workload_output=NESTED_OUTPUT + ), LegacyEvaluationCriteria( expected_output=NESTED_OUTPUT, expected_agent_behavior="" ), @@ -87,7 +89,9 @@ async def test_nested_dot_path_customer_address_city(self) -> None: **_make_exact_match_params("customer.address.city") ) result = await evaluator.evaluate( - AgentExecution(agent_input={}, agent_trace=[], agent_output=NESTED_OUTPUT), + WorkloadExecution( + agent_input={}, workload_trace=[], workload_output=NESTED_OUTPUT + ), LegacyEvaluationCriteria( expected_output=NESTED_OUTPUT, expected_agent_behavior="" ), @@ -100,7 +104,9 @@ async def test_array_index_items_0_name(self) -> None: **_make_exact_match_params("items[0].name") ) result = await evaluator.evaluate( - AgentExecution(agent_input={}, agent_trace=[], agent_output=NESTED_OUTPUT), + WorkloadExecution( + agent_input={}, workload_trace=[], workload_output=NESTED_OUTPUT + ), LegacyEvaluationCriteria( expected_output=NESTED_OUTPUT, expected_agent_behavior="" ), @@ -111,7 +117,9 @@ async def test_array_index_items_0_name(self) -> None: async def test_array_index_tags_1(self) -> None: evaluator = LegacyExactMatchEvaluator(**_make_exact_match_params("tags[1]")) result = await evaluator.evaluate( - AgentExecution(agent_input={}, agent_trace=[], agent_output=NESTED_OUTPUT), + WorkloadExecution( + agent_input={}, workload_trace=[], workload_output=NESTED_OUTPUT + ), LegacyEvaluationCriteria( expected_output=NESTED_OUTPUT, expected_agent_behavior="" ), @@ -128,8 +136,8 @@ async def test_nested_path_mismatch_fails(self) -> None: modified_summary = {**original_summary, "status": "pending"} different_output = {**NESTED_OUTPUT, "summary": modified_summary} result = await evaluator.evaluate( - AgentExecution( - agent_input={}, agent_trace=[], agent_output=different_output + WorkloadExecution( + agent_input={}, workload_trace=[], workload_output=different_output ), LegacyEvaluationCriteria( expected_output=NESTED_OUTPUT, expected_agent_behavior="" @@ -144,7 +152,9 @@ async def test_missing_path_in_both_passes(self) -> None: **_make_exact_match_params("nonexistent.path") ) result = await evaluator.evaluate( - AgentExecution(agent_input={}, agent_trace=[], agent_output=NESTED_OUTPUT), + WorkloadExecution( + agent_input={}, workload_trace=[], workload_output=NESTED_OUTPUT + ), LegacyEvaluationCriteria( expected_output=NESTED_OUTPUT, expected_agent_behavior="" ), @@ -156,7 +166,9 @@ async def test_flat_key_backward_compatible(self) -> None: """Flat key like 'order_id' still works as before.""" evaluator = LegacyExactMatchEvaluator(**_make_exact_match_params("order_id")) result = await evaluator.evaluate( - AgentExecution(agent_input={}, agent_trace=[], agent_output=NESTED_OUTPUT), + WorkloadExecution( + agent_input={}, workload_trace=[], workload_output=NESTED_OUTPUT + ), LegacyEvaluationCriteria( expected_output=NESTED_OUTPUT, expected_agent_behavior="" ), @@ -172,7 +184,9 @@ async def test_json_similarity_with_target_key_summary(self) -> None: """JSON similarity on nested 'summary' object should score 100.""" evaluator = LegacyJsonSimilarityEvaluator(**_make_json_sim_params("summary")) result = await evaluator.evaluate( - AgentExecution(agent_input={}, agent_trace=[], agent_output=NESTED_OUTPUT), + WorkloadExecution( + agent_input={}, workload_trace=[], workload_output=NESTED_OUTPUT + ), LegacyEvaluationCriteria( expected_output=NESTED_OUTPUT, expected_agent_behavior="" ), @@ -184,7 +198,9 @@ async def test_json_similarity_with_target_key_customer(self) -> None: """JSON similarity on nested 'customer' object should score 100.""" evaluator = LegacyJsonSimilarityEvaluator(**_make_json_sim_params("customer")) result = await evaluator.evaluate( - AgentExecution(agent_input={}, agent_trace=[], agent_output=NESTED_OUTPUT), + WorkloadExecution( + agent_input={}, workload_trace=[], workload_output=NESTED_OUTPUT + ), LegacyEvaluationCriteria( expected_output=NESTED_OUTPUT, expected_agent_behavior="" ), @@ -196,7 +212,9 @@ async def test_json_similarity_with_target_key_items_0(self) -> None: """JSON similarity on items[0] should score 100 when matching.""" evaluator = LegacyJsonSimilarityEvaluator(**_make_json_sim_params("items[0]")) result = await evaluator.evaluate( - AgentExecution(agent_input={}, agent_trace=[], agent_output=NESTED_OUTPUT), + WorkloadExecution( + agent_input={}, workload_trace=[], workload_output=NESTED_OUTPUT + ), LegacyEvaluationCriteria( expected_output=NESTED_OUTPUT, expected_agent_behavior="" ), @@ -208,7 +226,9 @@ async def test_json_similarity_wildcard_unchanged(self) -> None: """Wildcard '*' should compare the full output (backward compatible).""" evaluator = LegacyJsonSimilarityEvaluator(**_make_json_sim_params("*")) result = await evaluator.evaluate( - AgentExecution(agent_input={}, agent_trace=[], agent_output=NESTED_OUTPUT), + WorkloadExecution( + agent_input={}, workload_trace=[], workload_output=NESTED_OUTPUT + ), LegacyEvaluationCriteria( expected_output=NESTED_OUTPUT, expected_agent_behavior="" ), @@ -224,8 +244,8 @@ async def test_json_similarity_partial_match_with_target(self) -> None: "summary": {"total": 45.0, "item_count": 3, "status": "completed"}, } result = await evaluator.evaluate( - AgentExecution( - agent_input={}, agent_trace=[], agent_output=different_summary + WorkloadExecution( + agent_input={}, workload_trace=[], workload_output=different_summary ), LegacyEvaluationCriteria( expected_output=NESTED_OUTPUT, expected_agent_behavior="" diff --git a/packages/uipath/tests/evaluators/test_legacy_trajectory_evaluator.py b/packages/uipath/tests/evaluators/test_legacy_trajectory_evaluator.py new file mode 100644 index 000000000..687fce08d --- /dev/null +++ b/packages/uipath/tests/evaluators/test_legacy_trajectory_evaluator.py @@ -0,0 +1,64 @@ +import uuid + +from opentelemetry.sdk.trace import ReadableSpan + +from uipath.eval.evaluators import LegacyTrajectoryEvaluator +from uipath.eval.evaluators.base_legacy_evaluator import LegacyEvaluationCriteria +from uipath.eval.evaluators.legacy_trajectory_evaluator import ( + LegacyTrajectoryEvaluatorConfig, +) +from uipath.eval.models.models import LegacyEvaluatorCategory, LegacyEvaluatorType + + +def _legacy_trajectory_evaluator() -> LegacyTrajectoryEvaluator: + return LegacyTrajectoryEvaluator( + id=str(uuid.uuid4()), + name="Legacy trajectory", + config_type=LegacyTrajectoryEvaluatorConfig, + evaluation_criteria_type=LegacyEvaluationCriteria, + justification_type=str, + category=LegacyEvaluatorCategory.Trajectory, + type=LegacyEvaluatorType.Trajectory, + prompt="History:\n{{AgentRunHistory}}\nExpected:\n{{ExpectedAgentBehavior}}", + createdAt="2026-05-14T00:00:00Z", + updatedAt="2026-05-14T00:00:00Z", + ) + + +def test_legacy_trajectory_prompt_uses_compact_tool_history() -> None: + long_prompt = "SYSTEM_PROMPT_" + ("x" * 10_000) + spans = [ + ReadableSpan( + name="agent_llm_call", + start_time=0, + end_time=1, + attributes={ + "openinference.span.kind": "LLM", + "input.value": f'{{"messages": [{{"role": "system", "content": "{long_prompt}"}}]}}', + "output.value": '{"generations": []}', + }, + ), + ReadableSpan( + name="search_profiles", + start_time=1, + end_time=2, + attributes={ + "openinference.span.kind": "TOOL", + "tool.name": "search_profiles", + "input.value": '{"query": "mentor"}', + "output.value": '{"content": "found mentor profile"}', + "metadata": f'{{"agent_prompt": "{long_prompt}"}}', + }, + ), + ] + + prompt = _legacy_trajectory_evaluator()._create_evaluation_prompt( + expected_agent_behavior="The agent should search matching profiles.", + agent_run_history=spans, + ) + + assert "SYSTEM_PROMPT_" not in prompt + assert "Tool: search_profiles" in prompt + assert '{"query": "mentor"}' in prompt + assert "found mentor profile" in prompt + assert "agent_llm_call" not in prompt diff --git a/packages/uipath/tests/evaluators/test_line_by_line_utils.py b/packages/uipath/tests/evaluators/test_line_by_line_utils.py index 89449600c..dcf06f5f0 100644 --- a/packages/uipath/tests/evaluators/test_line_by_line_utils.py +++ b/packages/uipath/tests/evaluators/test_line_by_line_utils.py @@ -13,7 +13,7 @@ LineByLineEvaluationDetails, LineEvaluationDetail, ) -from uipath.eval.models.models import AgentExecution, NumericEvaluationResult +from uipath.eval.models.models import NumericEvaluationResult, WorkloadExecution class TestSplitIntoLines: @@ -154,10 +154,10 @@ async def test_evaluate_lines_all_match(self): actual_lines = ["line1", "line2", "line3"] expected_lines = ["line1", "line2", "line3"] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"test": "input"}, - agent_output="line1\nline2\nline3", - agent_trace=[], + workload_output="line1\nline2\nline3", + workload_trace=[], ) async def mock_evaluate(execution, criteria): @@ -170,7 +170,7 @@ def mock_create_criteria(expected_line): actual_lines=actual_lines, expected_lines=expected_lines, target_output_key="*", - agent_execution=agent_execution, + workload_execution=workload_execution, evaluate_fn=mock_evaluate, create_line_criteria_fn=mock_create_criteria, ) @@ -186,10 +186,10 @@ async def test_evaluate_lines_unequal_counts(self): actual_lines = ["line1", "line2"] expected_lines = ["line1", "line2", "line3", "line4"] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"test": "input"}, - agent_output="line1\nline2", - agent_trace=[], + workload_output="line1\nline2", + workload_trace=[], ) async def mock_evaluate(execution, criteria): @@ -202,7 +202,7 @@ def mock_create_criteria(expected_line): actual_lines=actual_lines, expected_lines=expected_lines, target_output_key="*", - agent_execution=agent_execution, + workload_execution=workload_execution, evaluate_fn=mock_evaluate, create_line_criteria_fn=mock_create_criteria, ) @@ -229,15 +229,15 @@ async def test_evaluate_lines_with_target_output_key(self): actual_lines = ["line1", "line2"] expected_lines = ["line1", "line2"] - agent_execution = AgentExecution( + workload_execution = WorkloadExecution( agent_input={"test": "input"}, - agent_output={"result": "line1\nline2"}, - agent_trace=[], + workload_output={"result": "line1\nline2"}, + workload_trace=[], ) async def mock_evaluate(execution, criteria): # Verify the execution has the wrapped structure - assert "result" in execution.agent_output + assert "result" in execution.workload_output return NumericEvaluationResult(score=1.0, details="match") def mock_create_criteria(expected_line): @@ -247,7 +247,7 @@ def mock_create_criteria(expected_line): actual_lines=actual_lines, expected_lines=expected_lines, target_output_key="result", - agent_execution=agent_execution, + workload_execution=workload_execution, evaluate_fn=mock_evaluate, create_line_criteria_fn=mock_create_criteria, ) diff --git a/packages/uipath/tests/evaluators/test_llm_judge_model_suffix.py b/packages/uipath/tests/evaluators/test_llm_judge_model_suffix.py new file mode 100644 index 000000000..de533a963 --- /dev/null +++ b/packages/uipath/tests/evaluators/test_llm_judge_model_suffix.py @@ -0,0 +1,219 @@ +"""Regression tests: LLM-judge evaluators must send the model name to the LLM +Gateway exactly as configured, including a "-community-agents" suffix. + +Community/EU tenants' LLM Gateway routing rules are keyed on the suffixed +model id -- the same id AgentHub sends when it runs the agent itself. +Stripping the suffix before calling the Gateway causes a 417 "No llm routing +rule found for product agentsplaygroundfallback in EU using model ..." for +every Community/EU evaluation run, even though the identical model id works +fine for the agent. +""" + +import uuid +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest + +from uipath.eval.evaluators.base_legacy_evaluator import LegacyEvaluationCriteria +from uipath.eval.evaluators.legacy_context_precision_evaluator import ( + LegacyContextPrecisionEvaluator, + LegacyContextPrecisionEvaluatorConfig, +) +from uipath.eval.evaluators.legacy_faithfulness_evaluator import ( + LegacyFaithfulnessEvaluator, + LegacyFaithfulnessEvaluatorConfig, +) +from uipath.eval.evaluators.legacy_llm_as_judge_evaluator import ( + LegacyLlmAsAJudgeEvaluator, + LegacyLlmAsAJudgeEvaluatorConfig, +) +from uipath.eval.evaluators.legacy_trajectory_evaluator import ( + LegacyTrajectoryEvaluator, + LegacyTrajectoryEvaluatorConfig, +) +from uipath.eval.evaluators.llm_judge_output_evaluator import LLMJudgeOutputEvaluator +from uipath.eval.evaluators.llm_judge_trajectory_evaluator import ( + LLMJudgeTrajectoryEvaluator, +) +from uipath.eval.models.models import LegacyEvaluatorCategory, LegacyEvaluatorType + +COMMUNITY_MODEL = "gpt-5.4-2026-03-05-community-agents" + + +def _fake_tool_call_response(score: float = 90, justification: str = "ok"): + tool_call = SimpleNamespace( + arguments={"score": score, "justification": justification} + ) + message = SimpleNamespace(tool_calls=[tool_call]) + choice = SimpleNamespace(message=message) + return SimpleNamespace(choices=[choice]) + + +def _legacy_context_precision_evaluator() -> LegacyContextPrecisionEvaluator: + return LegacyContextPrecisionEvaluator( + id="context-precision", + config_type=LegacyContextPrecisionEvaluatorConfig, + evaluation_criteria_type=LegacyEvaluationCriteria, + justification_type=str, + category=LegacyEvaluatorCategory.LlmAsAJudge, + type=LegacyEvaluatorType.ContextPrecision, + name="Context Precision", + description="Evaluates context chunk relevance", + createdAt="2025-01-01T00:00:00Z", + updatedAt="2025-01-01T00:00:00Z", + targetOutputKey="*", + model=COMMUNITY_MODEL, + ) + + +class TestLegacyContextPrecisionEvaluatorSendsConfiguredModel: + @pytest.mark.asyncio + async def test_get_structured_llm_response_sends_full_model_name(self): + evaluator = _legacy_context_precision_evaluator() + mock_chat_completions = AsyncMock(return_value=_fake_tool_call_response()) + evaluator.llm = AsyncMock(chat_completions=mock_chat_completions) + + await evaluator._get_structured_llm_response("some evaluation prompt") + + sent_model = mock_chat_completions.call_args.kwargs["model"] + assert sent_model == COMMUNITY_MODEL + + +def _legacy_faithfulness_evaluator() -> LegacyFaithfulnessEvaluator: + return LegacyFaithfulnessEvaluator( + id="faithfulness", + config_type=LegacyFaithfulnessEvaluatorConfig, + evaluation_criteria_type=LegacyEvaluationCriteria, + justification_type=str, + category=LegacyEvaluatorCategory.LlmAsAJudge, + type=LegacyEvaluatorType.Faithfulness, + name="Faithfulness", + description="Evaluates faithfulness of claims against context", + createdAt="2025-01-01T00:00:00Z", + updatedAt="2025-01-01T00:00:00Z", + targetOutputKey="*", + model=COMMUNITY_MODEL, + ) + + +class TestLegacyFaithfulnessEvaluatorSendsConfiguredModel: + @pytest.mark.asyncio + async def test_get_structured_llm_response_sends_full_model_name(self): + evaluator = _legacy_faithfulness_evaluator() + mock_chat_completions = AsyncMock(return_value=_fake_tool_call_response()) + evaluator.llm = AsyncMock(chat_completions=mock_chat_completions) + + await evaluator._get_structured_llm_response( + "some evaluation prompt", "submit_result", {"type": "object"} + ) + + sent_model = mock_chat_completions.call_args.kwargs["model"] + assert sent_model == COMMUNITY_MODEL + + +class TestLLMJudgeOutputEvaluatorSendsConfiguredModel: + """Covers LLMJudgeMixin._get_llm_response -- the code path hit by + 'uipath-llm-judge-output-semantic-similarity' in production.""" + + @pytest.mark.asyncio + async def test_get_llm_response_sends_full_model_name_to_gateway(self): + config = { + "name": "TestEvaluator", + "prompt": "Evaluate {{ActualOutput}} against {{ExpectedOutput}}", + "model": COMMUNITY_MODEL, + } + with patch("uipath.platform.UiPath"): + evaluator = LLMJudgeOutputEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + mock_llm_service = AsyncMock(return_value=_fake_tool_call_response()) + evaluator.llm_service = mock_llm_service + + await evaluator._get_llm_response("some evaluation prompt") + + sent_model = mock_llm_service.call_args.kwargs["model"] + assert sent_model == COMMUNITY_MODEL + + +class TestLLMJudgeTrajectoryEvaluatorSendsConfiguredModel: + """Covers the same LLMJudgeMixin._get_llm_response via the trajectory + evaluator -- this is the exact evaluator type from the reported + production trace ('uipath-llm-judge-trajectory-similarity').""" + + @pytest.mark.asyncio + async def test_get_llm_response_sends_full_model_name_to_gateway(self): + config = { + "name": "TestEvaluator", + "prompt": "Judge {{AgentRunHistory}} against {{ExpectedAgentBehavior}}", + "model": COMMUNITY_MODEL, + } + with patch("uipath.platform.UiPath"): + evaluator = LLMJudgeTrajectoryEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + mock_llm_service = AsyncMock(return_value=_fake_tool_call_response()) + evaluator.llm_service = mock_llm_service + + await evaluator._get_llm_response("some evaluation prompt") + + sent_model = mock_llm_service.call_args.kwargs["model"] + assert sent_model == COMMUNITY_MODEL + + +def _legacy_trajectory_evaluator() -> LegacyTrajectoryEvaluator: + return LegacyTrajectoryEvaluator( + id=str(uuid.uuid4()), + name="Legacy trajectory", + config_type=LegacyTrajectoryEvaluatorConfig, + evaluation_criteria_type=LegacyEvaluationCriteria, + justification_type=str, + category=LegacyEvaluatorCategory.Trajectory, + type=LegacyEvaluatorType.Trajectory, + prompt="History:\n{{AgentRunHistory}}\nExpected:\n{{ExpectedAgentBehavior}}", + model=COMMUNITY_MODEL, + createdAt="2026-05-14T00:00:00Z", + updatedAt="2026-05-14T00:00:00Z", + ) + + +class TestLegacyTrajectoryEvaluatorSendsConfiguredModel: + @pytest.mark.asyncio + async def test_get_llm_response_sends_full_model_name_to_gateway(self): + evaluator = _legacy_trajectory_evaluator() + mock_chat_completions = AsyncMock(return_value=_fake_tool_call_response()) + evaluator.llm = AsyncMock(chat_completions=mock_chat_completions) + + await evaluator._get_llm_response("some evaluation prompt") + + sent_model = mock_chat_completions.call_args.kwargs["model"] + assert sent_model == COMMUNITY_MODEL + + +def _legacy_llm_as_judge_evaluator() -> LegacyLlmAsAJudgeEvaluator: + return LegacyLlmAsAJudgeEvaluator( + id=str(uuid.uuid4()), + name="Legacy LLM judge", + config_type=LegacyLlmAsAJudgeEvaluatorConfig, + evaluation_criteria_type=LegacyEvaluationCriteria, + justification_type=str, + category=LegacyEvaluatorCategory.LlmAsAJudge, + type=LegacyEvaluatorType.Factuality, + prompt="Compare {{ActualOutput}} to {{ExpectedOutput}}", + model=COMMUNITY_MODEL, + createdAt="2026-05-14T00:00:00Z", + updatedAt="2026-05-14T00:00:00Z", + ) + + +class TestLegacyLlmAsAJudgeEvaluatorSendsConfiguredModel: + @pytest.mark.asyncio + async def test_get_llm_response_sends_full_model_name_to_gateway(self): + evaluator = _legacy_llm_as_judge_evaluator() + mock_chat_completions = AsyncMock(return_value=_fake_tool_call_response()) + evaluator.llm = AsyncMock(chat_completions=mock_chat_completions) + + await evaluator._get_llm_response("some evaluation prompt") + + sent_model = mock_chat_completions.call_args.kwargs["model"] + assert sent_model == COMMUNITY_MODEL diff --git a/packages/uipath/tests/evaluators/test_workload_execution_deprecation.py b/packages/uipath/tests/evaluators/test_workload_execution_deprecation.py new file mode 100644 index 000000000..98ccddcbf --- /dev/null +++ b/packages/uipath/tests/evaluators/test_workload_execution_deprecation.py @@ -0,0 +1,87 @@ +"""Tests for the ``AgentExecution`` -> ``WorkloadExecution`` rename (v2.12.0). + +Covers the soft-deprecated class-name shim, the intentional hard break on the +renamed model fields, and the positional-dispatch contract that keeps custom +evaluators using the old ``agent_execution`` parameter name working. +""" + +import importlib +import uuid + +import pytest +from pydantic import ValidationError + +from uipath.eval.evaluators.exact_match_evaluator import ExactMatchEvaluator +from uipath.eval.models import NumericEvaluationResult, WorkloadExecution + + +@pytest.mark.parametrize( + "module_path", + ["uipath.eval.models", "uipath.eval.models.models"], +) +def test_agent_execution_alias_warns_and_resolves(module_path: str) -> None: + """Accessing the legacy ``AgentExecution`` name warns and returns the new class.""" + module = importlib.import_module(module_path) + + with pytest.warns(DeprecationWarning, match="AgentExecution is deprecated"): + legacy = module.AgentExecution + + assert legacy is WorkloadExecution + + +@pytest.mark.parametrize( + "module_path", + ["uipath.eval.models", "uipath.eval.models.models"], +) +def test_unknown_attribute_raises(module_path: str) -> None: + """Unknown attributes still raise ``AttributeError`` via the module ``__getattr__``.""" + module = importlib.import_module(module_path) + + with pytest.raises(AttributeError, match="does_not_exist"): + _ = module.does_not_exist + + +def test_old_field_names_are_a_hard_break() -> None: + """The renamed fields are NOT aliased — old field names raise ValidationError. + + This is the intentional breaking change in v2.12.0 (no field-level back-compat + shim); only the class *name* is soft-deprecated. + """ + with pytest.raises(ValidationError): + WorkloadExecution( + agent_input={}, + agent_output={"result": "ok"}, # type: ignore[call-arg] + agent_trace=[], + ) + + +async def test_old_param_name_still_dispatches_positionally() -> None: + """A custom evaluator overriding ``evaluate`` with the old ``agent_execution`` + parameter name keeps working, because the base dispatches positionally. + """ + + class CustomEvaluator(ExactMatchEvaluator): + # Deliberately uses the pre-2.12.0 parameter name. + async def evaluate(self, agent_execution, evaluation_criteria): + return await super().evaluate(agent_execution, evaluation_criteria) + + evaluator = CustomEvaluator.model_validate( + { + "evaluatorConfig": {"name": "CustomExactMatch", "case_sensitive": True}, + "id": str(uuid.uuid4()), + } + ) + workload_execution = WorkloadExecution( + agent_input={}, + workload_output={"output": "Test output"}, + workload_trace=[], + ) + raw_criteria = {"expected_output": {"output": "Test output"}} + + # Called positionally (as the runtime does) — must not raise TypeError. + result = await evaluator.validate_and_evaluate_criteria( + workload_execution, raw_criteria + ) + + assert isinstance(result, NumericEvaluationResult) + assert result.score == 1.0 diff --git a/packages/uipath/tests/functions/test_input_validation.py b/packages/uipath/tests/functions/test_input_validation.py new file mode 100644 index 000000000..e76ca922e --- /dev/null +++ b/packages/uipath/tests/functions/test_input_validation.py @@ -0,0 +1,106 @@ +"""Tests that invalid inputs surface as classified user errors, not crashes.""" + +import textwrap + +import pytest + +from uipath.functions.runtime import UiPathFunctionsRuntime +from uipath.runtime.errors import UiPathErrorCategory, UiPathRuntimeError + + +@pytest.fixture +def calculator_module(tmp_path): + """Create a module with a Pydantic-typed entrypoint.""" + (tmp_path / "calculator.py").write_text( + textwrap.dedent("""\ + from pydantic import BaseModel + + + class CalculatorInput(BaseModel): + a: int + b: int + + + class CalculatorOutput(BaseModel): + result: int + + + async def main(input: CalculatorInput) -> CalculatorOutput: + return CalculatorOutput(result=input.a + input.b) + """) + ) + return tmp_path / "calculator.py" + + +@pytest.mark.asyncio +async def test_valid_input_executes(calculator_module): + """Sanity check: well-formed input still executes normally.""" + runtime = UiPathFunctionsRuntime(str(calculator_module), "main", "calculator") + result = await runtime.execute({"a": 1, "b": 2}) + assert result.output == {"result": 3} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "bad_input", + [ + {"a": "hello", "b": 2}, + {"a": None, "b": 2}, + {"a": [1, 2], "b": 2}, + {"a": {"x": 1}, "b": 2}, + ], +) +async def test_invalid_input_raises_user_error(calculator_module, bad_input): + """Schema-mismatched input yields a USER-category error without a traceback.""" + runtime = UiPathFunctionsRuntime(str(calculator_module), "main", "calculator") + with pytest.raises(UiPathRuntimeError) as exc_info: + await runtime.execute(bad_input) + + error_info = exc_info.value.error_info + assert error_info.category == UiPathErrorCategory.USER + assert error_info.code == "Python.INPUT_INVALID_JSON" + assert error_info.title == "Invalid input" + assert "CalculatorInput" in error_info.detail + assert "main" in error_info.detail + assert "Traceback" not in error_info.detail + + +@pytest.mark.asyncio +async def test_missing_dataclass_field_raises_user_error(tmp_path): + """Non-Pydantic conversion failures (TypeError) are classified the same way.""" + (tmp_path / "shipping.py").write_text( + textwrap.dedent("""\ + from dataclasses import dataclass + + + @dataclass + class ShippingInput: + address: str + zip_code: str + + + async def main(input: ShippingInput) -> dict: + return {"ok": True} + """) + ) + runtime = UiPathFunctionsRuntime(str(tmp_path / "shipping.py"), "main", "shipping") + with pytest.raises(UiPathRuntimeError) as exc_info: + await runtime.execute({"address": "1 Main St"}) + + error_info = exc_info.value.error_info + assert error_info.category == UiPathErrorCategory.USER + assert error_info.title == "Invalid input" + assert "ShippingInput" in error_info.detail + assert "zip_code" in error_info.detail + + +@pytest.mark.asyncio +async def test_invalid_input_error_lists_offending_fields(calculator_module): + """The error detail names each invalid field with the validation message.""" + runtime = UiPathFunctionsRuntime(str(calculator_module), "main", "calculator") + with pytest.raises(UiPathRuntimeError) as exc_info: + await runtime.execute({"a": "hello", "b": "world"}) + + detail = exc_info.value.error_info.detail + assert "a:" in detail + assert "b:" in detail diff --git a/packages/uipath/tests/resource_overrides/overwrites.json b/packages/uipath/tests/resource_overrides/overwrites.json index c58744a69..e0bca84ba 100644 --- a/packages/uipath/tests/resource_overrides/overwrites.json +++ b/packages/uipath/tests/resource_overrides/overwrites.json @@ -28,5 +28,9 @@ "mcpServer.mcp_server_name": { "name": "Overwritten MCP Server Name", "folderPath": "Overwritten/MCPServer/Folder" + }, + "entity.entity_name": { + "name": "Overwritten Entity Name", + "folderId": "overwritten-entity-folder-id-123" } } \ No newline at end of file diff --git a/packages/uipath/tests/resource_overrides/test_overwrites_logging.py b/packages/uipath/tests/resource_overrides/test_overwrites_logging.py new file mode 100644 index 000000000..e05a7daed --- /dev/null +++ b/packages/uipath/tests/resource_overrides/test_overwrites_logging.py @@ -0,0 +1,308 @@ +# type: ignore +"""Tests for INFO-level diagnostic logging on the resource-overwrites read paths. + +Covers the recent change that surfaces bindings.json content and raw resource +overwrites (from both uipath.json and the Studio API) at INFO so binding/ +overwrite mismatches can be diagnosed from logs alone. +""" + +import json +import logging +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from uipath._cli._utils._common import read_resource_overwrites_from_file +from uipath._cli._utils._studio_project import StudioClient +from uipath.platform.common import GenericResourceOverwrite + +_VALID_OVERWRITES = { + "asset.asset_name": { + "name": "Overwritten Asset Name", + "folderPath": "Overwritten/Asset/Folder", + }, + "bucket.bucket_name": { + "name": "Overwritten Bucket Name", + "folderPath": "Overwritten/Bucket/Folder", + }, +} + + +_TARGET_LOGGERS = ( + "uipath._cli._utils._common", + "uipath._cli._utils._studio_project", +) + + +@pytest.fixture(autouse=True) +def _capture_uipath_loggers( + caplog: pytest.LogCaptureFixture, +) -> None: + """Attach caplog's handler directly to the target module loggers. + + Earlier tests in the suite — chiefly anything that invokes the Click CLI + — call ``setup_logging`` and leave the ``uipath`` logger with + ``propagate = False``. That breaks the usual caplog flow (handler on + root, records reach it via propagation). Some intermediate loggers can + also end up with ``propagate = False`` from other test setups. Attaching + the handler directly to each module logger we assert against, and + forcing the level to DEBUG for the duration of the test, side-steps the + propagation question entirely. + """ + snapshots: list[tuple[logging.Logger, int, bool]] = [] + for name in _TARGET_LOGGERS: + logger = logging.getLogger(name) + snapshots.append((logger, logger.level, logger.propagate)) + logger.setLevel(logging.DEBUG) + logger.propagate = True + logger.addHandler(caplog.handler) + try: + yield + finally: + for logger, level, propagate in snapshots: + logger.removeHandler(caplog.handler) + logger.setLevel(level) + logger.propagate = propagate + + +def _write_uipath_json(directory: Path, overwrites: dict) -> Path: + config_path = directory / "uipath.json" + config_path.write_text( + json.dumps( + { + "runtime": {"internalArguments": {"resourceOverwrites": overwrites}}, + } + ) + ) + return config_path + + +class TestReadResourceOverwritesFromFileLogging: + """Behavior: read_resource_overwrites_from_file logs diagnostic info at INFO.""" + + async def test_logs_raw_overwrites_at_info_when_file_present( + self, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + config_path = _write_uipath_json(tmp_path, _VALID_OVERWRITES) + + with caplog.at_level(logging.INFO, logger="uipath._cli._utils._common"): + result = await read_resource_overwrites_from_file(str(tmp_path)) + + assert set(result.keys()) == set(_VALID_OVERWRITES.keys()) + + info_records = [r for r in caplog.records if r.levelno == logging.INFO] + assert any( + "Resource overwrites read from" in r.getMessage() + and str(config_path) in r.getMessage() + and f"({len(_VALID_OVERWRITES)} entries)" in r.getMessage() + for r in info_records + ), f"expected INFO log with file path and entry count, got: {caplog.text}" + + # The raw JSON payload should be present in the log so a developer can + # diff it against what Studio later returns. + assert "Overwritten Asset Name" in caplog.text + assert "Overwritten Bucket Name" in caplog.text + + async def test_logs_info_when_config_file_missing( + self, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + # tmp_path is empty — no uipath.json present. + missing_dir = tmp_path / "does-not-exist" + missing_dir.mkdir() + + with caplog.at_level(logging.INFO, logger="uipath._cli._utils._common"): + result = await read_resource_overwrites_from_file(str(missing_dir)) + + assert result == {} + info_messages = [ + r.getMessage() for r in caplog.records if r.levelno == logging.INFO + ] + assert any( + "Resource overwrites config file not found" in msg for msg in info_messages + ), f"expected INFO log for missing config, got: {info_messages}" + + async def test_logs_warning_when_json_is_malformed( + self, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + (tmp_path / "uipath.json").write_text("{not valid json") + + with caplog.at_level(logging.WARNING, logger="uipath._cli._utils._common"): + result = await read_resource_overwrites_from_file(str(tmp_path)) + + assert result == {} + warnings = [r for r in caplog.records if r.levelno == logging.WARNING] + assert any( + "Failed to parse resource overwrites" in r.getMessage() for r in warnings + ) + + async def test_unrecognized_overwrite_key_is_skipped_with_warning( + self, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + overwrites = { + **_VALID_OVERWRITES, + "totallyUnknownKind.foo": {"name": "x", "folderPath": "y"}, + } + _write_uipath_json(tmp_path, overwrites) + + with caplog.at_level(logging.WARNING, logger="uipath._cli._utils._common"): + result = await read_resource_overwrites_from_file(str(tmp_path)) + + # Valid entries still parsed; unknown key dropped. + assert set(result.keys()) == set(_VALID_OVERWRITES.keys()) + assert any( + "Skipping unrecognized resource overwrite" in r.getMessage() + and "totallyUnknownKind.foo" in r.getMessage() + for r in caplog.records + if r.levelno == logging.WARNING + ) + + +class TestStudioClientGetResourceOverwritesLogging: + """Behavior: StudioClient.get_resource_overwrites logs bindings + raw payload.""" + + @pytest.fixture + def studio_client(self) -> StudioClient: + # Inject a mock UiPath so no real HTTP setup is required. + mock_uipath = MagicMock() + mock_uipath.api_client.request_async = AsyncMock() + client = StudioClient(project_id="test-project-id", uipath=mock_uipath) + # Avoid the network call that resolves the solution id. + client._get_solution_id = AsyncMock(return_value="test-solution-id") # type: ignore[method-assign] + return client + + async def test_warns_and_returns_empty_when_bindings_file_missing( + self, + studio_client: StudioClient, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + ) -> None: + from uipath.platform.common._config import ConfigurationManager + + missing_path = tmp_path / "bindings.json" + monkeypatch.setattr( + ConfigurationManager, + "bindings_file_path", + property(lambda self: missing_path), + ) + + with caplog.at_level(logging.WARNING): + result = await studio_client.get_resource_overwrites() + + assert result == {} + assert any( + "Bindings file not found" in r.getMessage() + for r in caplog.records + if r.levelno == logging.WARNING + ) + # No request should have been made when there is nothing to upload. + studio_client.uipath.api_client.request_async.assert_not_called() + + async def test_logs_bindings_content_and_received_overwrites_at_info( + self, + studio_client: StudioClient, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + ) -> None: + from uipath.platform.common._config import ConfigurationManager + + bindings_path = tmp_path / "bindings.json" + bindings_content = json.dumps( + {"version": "2", "resources": [{"name": "my_bucket", "kind": "bucket"}]} + ) + bindings_path.write_text(bindings_content) + monkeypatch.setattr( + ConfigurationManager, + "bindings_file_path", + property(lambda self: bindings_path), + ) + monkeypatch.delenv("UIPATH_TENANT_ID", raising=False) + + response = MagicMock() + response.json.return_value = { + "bucket.my_bucket": { + "name": "prod_bucket", + "folderPath": "Prod/Folder", + } + } + studio_client.uipath.api_client.request_async = AsyncMock(return_value=response) + + with caplog.at_level(logging.INFO, logger="uipath._cli._utils._studio_project"): + result = await studio_client.get_resource_overwrites() + + # Returned dict is parsed via ResourceOverwriteParser. + assert set(result.keys()) == {"bucket.my_bucket"} + + info_text = "\n".join( + r.getMessage() for r in caplog.records if r.levelno == logging.INFO + ) + # Bindings content is logged so we can compare what was sent to Studio. + assert "Resource bindings" in info_text + assert "my_bucket" in info_text + # Received overwrites payload is logged with the solution id and count. + assert "Resource overwrites received for solution test-solution-id" in info_text + assert "(1 entries)" in info_text + assert "prod_bucket" in info_text + + async def test_parses_received_overwrites_into_resource_overwrite_objects( + self, + studio_client: StudioClient, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + from uipath.platform.common._config import ConfigurationManager + + bindings_path = tmp_path / "bindings.json" + bindings_path.write_text("{}") + monkeypatch.setattr( + ConfigurationManager, + "bindings_file_path", + property(lambda self: bindings_path), + ) + + response = MagicMock() + response.json.return_value = { + "bucket.my_bucket": { + "name": "prod_bucket", + "folderPath": "Prod/Folder", + } + } + studio_client.uipath.api_client.request_async = AsyncMock(return_value=response) + + result = await studio_client.get_resource_overwrites() + + parsed = result["bucket.my_bucket"] + assert isinstance(parsed, GenericResourceOverwrite) + assert parsed.resource_identifier == "prod_bucket" + assert parsed.folder_identifier == "Prod/Folder" + + async def test_passes_tenant_id_header_from_environment( + self, + studio_client: StudioClient, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + from uipath.platform.common._config import ConfigurationManager + + bindings_path = tmp_path / "bindings.json" + bindings_path.write_text("{}") + monkeypatch.setattr( + ConfigurationManager, + "bindings_file_path", + property(lambda self: bindings_path), + ) + monkeypatch.setenv("UIPATH_TENANT_ID", "tenant-from-env") + + response = MagicMock() + response.json.return_value = {} + request_mock = AsyncMock(return_value=response) + studio_client.uipath.api_client.request_async = request_mock + + await studio_client.get_resource_overwrites() + + # The header carrying the tenant id should reflect the env var value. + call_kwargs = request_mock.await_args.kwargs + headers = call_kwargs["headers"] + assert any(value == "tenant-from-env" for value in headers.values()), headers diff --git a/packages/uipath/tests/resource_overrides/test_resource_overrides.py b/packages/uipath/tests/resource_overrides/test_resource_overrides.py index c15bc113b..8d39a762d 100644 --- a/packages/uipath/tests/resource_overrides/test_resource_overrides.py +++ b/packages/uipath/tests/resource_overrides/test_resource_overrides.py @@ -310,6 +310,11 @@ def test_parse_overwrites_with_type_adapter(self, overwrites_data): assert mcp_server.resource_identifier == "Overwritten MCP Server Name" assert mcp_server.folder_identifier == "Overwritten/MCPServer/Folder" + # Verify entity overwrite + entity = parsed_overwrites["entity.entity_name"] + assert entity.resource_identifier == "Overwritten Entity Name" + assert entity.folder_identifier == "overwritten-entity-folder-id-123" + def test_overrides_decorator_should_pop_kwargs_dict_when_present(self): from uipath.platform.common import resource_override diff --git a/packages/uipath/tests/sdk/test_bindings.py b/packages/uipath/tests/sdk/test_bindings.py index d9afd8235..c581cb2ac 100644 --- a/packages/uipath/tests/sdk/test_bindings.py +++ b/packages/uipath/tests/sdk/test_bindings.py @@ -131,6 +131,58 @@ def dummy_func(name, folder_path): _resource_overwrites.reset(token) +class TestStackedResourceOverrideDecorators: + @pytest.mark.anyio + @pytest.mark.parametrize( + ("arguments", "expected"), + [ + ( + {"name": "original-resource"}, + (None, "replacement-resource", "replacement-folder"), + ), + ( + {"slug": "original-resource"}, + ("replacement-resource", None, "replacement-folder"), + ), + ], + ids=["outer-name-decorator", "inner-slug-decorator"], + ) + async def test_decorators_apply_override_for_active_identifier( + self, + arguments, + expected, + ): + """The outer name decorator must not prevent the inner slug override.""" + overwrite = GenericResourceOverwrite( + resource_type="mcpServer", + name="replacement-resource", + folder_path="replacement-folder", + ) + + @resource_override(resource_type="mcpServer", resource_identifier="name") + @resource_override(resource_type="mcpServer", resource_identifier="slug") + def retrieve(slug=None, *, name=None, folder_path=None): + return slug, name, folder_path + + @resource_override( + resource_type="mcpServer", + resource_identifier="name", + ) + @resource_override( + resource_type="mcpServer", + resource_identifier="slug", + ) + async def retrieve_async(slug=None, *, name=None, folder_path=None): + return slug, name, folder_path + + token = _resource_overwrites.set({"mcpServer.original-resource": overwrite}) + try: + assert retrieve(**arguments) == expected + assert await retrieve_async(**arguments) == expected + finally: + _resource_overwrites.reset(token) + + class TestResourceOverwritesContext: """Test that ResourceOverwritesContext works correctly with infer_bindings.""" @@ -391,3 +443,28 @@ def test_parse_connection_with_capitalized_alias(self): assert isinstance(overwrite, ConnectionResourceOverwrite) assert overwrite.connection_id == "conn-456" assert overwrite.folder_key == "folder2" + + +class TestRemoteA2aAgentResourceOverwrite: + """Test that Remote A2A agent resources parse as GenericResourceOverwrite.""" + + def test_remote_a2a_agent_resource_overwrite(self): + overwrite = GenericResourceOverwrite( + resource_type="remoteA2aAgent", + name="basica2a", + folder_path="Customers/ProjectA", + ) + assert overwrite.resource_type == "remoteA2aAgent" + assert overwrite.resource_identifier == "basica2a" + assert overwrite.folder_identifier == "Customers/ProjectA" + + def test_parse_remote_a2a_agent(self): + """Parser accepts a remoteA2aAgent-keyed overwrite without discriminator error.""" + overwrite = ResourceOverwriteParser.parse( + key="remoteA2aAgent.basica2a.solution_folder", + value={"name": "basica2a", "folderPath": "Customers/ProjectA"}, + ) + assert isinstance(overwrite, GenericResourceOverwrite) + assert overwrite.resource_type == "remoteA2aAgent" + assert overwrite.resource_identifier == "basica2a" + assert overwrite.folder_identifier == "Customers/ProjectA" diff --git a/packages/uipath/tests/sdk/test_utils_constants_shim.py b/packages/uipath/tests/sdk/test_utils_constants_shim.py new file mode 100644 index 000000000..a29334413 --- /dev/null +++ b/packages/uipath/tests/sdk/test_utils_constants_shim.py @@ -0,0 +1,76 @@ +"""Regression tests for the uipath._utils.constants deprecation shim. + +The shim re-exports from uipath.platform.constants (the source of truth) and emits +a FutureWarning so external consumers can migrate. Internal callsites are +already on the canonical path; these tests pin the shim's behavior so it keeps +working for downstream code. +""" + +import importlib +import sys +import warnings + + +def _reload_shim(): + """Force a fresh import of the shim so FutureWarning re-fires.""" + sys.modules.pop("uipath._utils.constants", None) + return importlib.import_module("uipath._utils.constants") + + +def test_shim_emits_future_warning(): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + _reload_shim() + + shim_warnings = [ + w + for w in caught + if issubclass(w.category, FutureWarning) + and "uipath._utils.constants" in str(w.message) + and "uipath.platform.constants" in str(w.message) + ] + assert len(shim_warnings) == 1, ( + f"expected exactly one shim FutureWarning, got {len(shim_warnings)}: " + f"{[str(w.message) for w in caught]}" + ) + + +def test_shim_re_exports_canonical_symbols(): + shim = _reload_shim() + canonical = importlib.import_module("uipath.platform.constants") + + # Sample a representative set: env vars, headers, mixed-case symbols, + # file constants, data-source magic strings. + sample = [ + "DOTENV_FILE", + "ENV_BASE_URL", + "ENV_TENANT_ID", + "HEADER_INTERNAL_TENANT_ID", + "HEADER_INTERNAL_ACCOUNT_ID", + "HEADER_USER_AGENT", + "LLMV3Mini_REQUEST", + "LLMV4_REQUEST", + "NativeV1_REQUEST", + "COMMUNITY_agents_SUFFIX", + "PYTHON_CONFIGURATION_FILE", + "ORCHESTRATOR_STORAGE_BUCKET_DATA_SOURCE_REQUEST", + ] + for name in sample: + assert hasattr(shim, name), f"shim missing {name}" + assert hasattr(canonical, name), f"canonical missing {name}" + assert getattr(shim, name) == getattr(canonical, name), ( + f"value drift for {name}: shim={getattr(shim, name)!r} " + f"canonical={getattr(canonical, name)!r}" + ) + + +def test_shim_does_not_leak_warnings_module_via_star_import(): + """The shim binds `warnings` under a private alias to keep it out of + `from uipath._utils.constants import *`.""" + _reload_shim() + ns: dict[str, object] = {} + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + exec("from uipath._utils.constants import *", ns) + assert "warnings" not in ns + assert "_warnings" not in ns diff --git a/packages/uipath/tests/telemetry/test_track.py b/packages/uipath/tests/telemetry/test_track.py index fe72130d5..738c4ea2b 100644 --- a/packages/uipath/tests/telemetry/test_track.py +++ b/packages/uipath/tests/telemetry/test_track.py @@ -1,11 +1,25 @@ """Tests for telemetry tracking functionality.""" +import json import os +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from queue import Empty, Queue from unittest.mock import MagicMock, patch +import pytest + +from uipath.core.feature_flags import FeatureFlags +from uipath.platform.constants import ( + ENV_PROJECT_KEY, + ENV_UIPATH_AGENT_ID, + ENV_UIPATH_PROJECT_ID, +) +from uipath.telemetry import PERIODIC_TELEMETRY_FLUSH_FEATURE_FLAG from uipath.telemetry._track import ( _AppInsightsEventClient, _DiagnosticSender, + _get_project_key, _parse_connection_string, _TelemetryClient, flush_events, @@ -17,6 +31,86 @@ ) +@pytest.fixture +def appinsights_ingestion_server(): + """Capture App Insights ingestion requests on a local HTTP server.""" + received: Queue[list[dict[str, object]]] = Queue() + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: + content_length = int(self.headers.get("Content-Length", "0")) + payload = json.loads(self.rfile.read(content_length)) + received.put(payload) + self.send_response(200) + self.end_headers() + + def log_message(self, format: str, *args: object) -> None: + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}", received + finally: + server.shutdown() + server.server_close() + server_thread.join() + + +def _event_names(payload: list[dict[str, object]]) -> list[str]: + """Extract custom-event names from an App Insights ingestion payload.""" + names: list[str] = [] + for envelope in payload: + data = envelope.get("data") + if not isinstance(data, dict): + continue + base_data = data.get("baseData") + if not isinstance(base_data, dict): + continue + name = base_data.get("name") + if isinstance(name, str): + names.append(name) + return names + + +class TestGetProjectKey: + """`_get_project_key` resolution: uipath.json#id, then legacy telemetry file.""" + + @pytest.fixture(autouse=True) + def _clear_cache(self, monkeypatch): + from uipath.platform.common._span_utils import _read_config_id + + _read_config_id.cache_clear() + for var in (ENV_UIPATH_AGENT_ID, ENV_UIPATH_PROJECT_ID, ENV_PROJECT_KEY): + monkeypatch.delenv(var, raising=False) + yield + _read_config_id.cache_clear() + + def test_prefers_uipath_json_id(self, monkeypatch, tmp_path): + config_id = "00000000-0000-0000-0000-000000000001" + (tmp_path / "uipath.json").write_text(json.dumps({"id": config_id})) + os.makedirs(tmp_path / ".uipath", exist_ok=True) + (tmp_path / ".uipath" / ".telemetry.json").write_text( + json.dumps({"ProjectKey": "from-telemetry"}) + ) + monkeypatch.chdir(tmp_path) + assert _get_project_key() == config_id + + def test_falls_back_to_legacy_telemetry_file(self, monkeypatch, tmp_path): + # No uipath.json#id and no env var; honor an existing .telemetry.json. + os.makedirs(tmp_path / ".uipath", exist_ok=True) + (tmp_path / ".uipath" / ".telemetry.json").write_text( + json.dumps({"ProjectKey": "from-telemetry"}) + ) + monkeypatch.chdir(tmp_path) + assert _get_project_key() == "from-telemetry" + + def test_unknown_when_no_source(self, monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + assert _get_project_key() == "" + + class TestParseConnectionString: """Test connection string parsing functionality.""" @@ -82,15 +176,15 @@ class TestAppInsightsEventClient: def setup_method(self): """Reset AppInsightsEventClient state before each test.""" - _AppInsightsEventClient._initialized = False - _AppInsightsEventClient._client = None + _AppInsightsEventClient.reset() _AppInsightsEventClient._connection_string_provider = None + FeatureFlags.reset_flags() def teardown_method(self): """Clean up after each test.""" - _AppInsightsEventClient._initialized = False - _AppInsightsEventClient._client = None + _AppInsightsEventClient.reset() _AppInsightsEventClient._connection_string_provider = None + FeatureFlags.reset_flags() @patch("uipath.telemetry._track._CONNECTION_STRING", "$CONNECTION_STRING") def test_initialize_no_connection_string(self): @@ -382,6 +476,196 @@ def test_reset_allows_reinitialization_with_new_connection_string( assert _AppInsightsEventClient._client is mock_client_2 assert mock_client_class.call_count == 2 + def test_single_event_remains_buffered_when_periodic_flush_disabled( + self, monkeypatch, appinsights_ingestion_server + ): + """Default behavior buffers one event until an explicit flush.""" + endpoint, received = appinsights_ingestion_server + monkeypatch.setenv( + "TELEMETRY_CONNECTION_STRING", + f"InstrumentationKey=test-key;IngestionEndpoint={endpoint}", + ) + monkeypatch.setattr( + "uipath.telemetry._track._PERIODIC_TELEMETRY_FLUSH_INTERVAL_SECONDS", + 0.05, + ) + + track_event("single-buffered-event") + + with pytest.raises(Empty): + received.get(timeout=0.15) + + flush_events() + payload = received.get(timeout=1) + assert _event_names(payload) == ["single-buffered-event"] + + def test_single_event_is_sent_by_periodic_flush( + self, monkeypatch, appinsights_ingestion_server + ): + """Enabled periodic flushing sends one event without an explicit flush.""" + endpoint, received = appinsights_ingestion_server + monkeypatch.setenv( + "TELEMETRY_CONNECTION_STRING", + f"InstrumentationKey=test-key;IngestionEndpoint={endpoint}", + ) + monkeypatch.setattr( + "uipath.telemetry._track._PERIODIC_TELEMETRY_FLUSH_INTERVAL_SECONDS", + 0.05, + ) + monkeypatch.setenv( + f"UIPATH_FEATURE_{PERIODIC_TELEMETRY_FLUSH_FEATURE_FLAG}", "true" + ) + + track_event("single-periodic-event") + + payload = received.get(timeout=1) + assert _event_names(payload) == ["single-periodic-event"] + + def test_periodic_worker_skips_flush_without_pending_events(self, monkeypatch): + """Periodic ticks do not flush while no event is queued.""" + monkeypatch.setattr( + "uipath.telemetry._track._PERIODIC_TELEMETRY_FLUSH_INTERVAL_SECONDS", + 0.01, + ) + FeatureFlags.configure_flags({PERIODIC_TELEMETRY_FLUSH_FEATURE_FLAG: True}) + mock_client = MagicMock() + monkeypatch.setattr(_AppInsightsEventClient, "_initialized", True) + monkeypatch.setattr(_AppInsightsEventClient, "_client", mock_client) + + _AppInsightsEventClient._ensure_periodic_flush_worker() + threading.Event().wait(0.05) + + mock_client.flush.assert_not_called() + + def test_shutdown_waits_for_worker_and_serializes_final_flush(self, monkeypatch): + """Shutdown joins an active worker before performing its final flush.""" + monkeypatch.setattr( + "uipath.telemetry._track._PERIODIC_TELEMETRY_FLUSH_INTERVAL_SECONDS", + 0.01, + ) + FeatureFlags.configure_flags({PERIODIC_TELEMETRY_FLUSH_FEATURE_FLAG: True}) + + flush_entered = threading.Event() + release_flush = threading.Event() + counter_lock = threading.Lock() + active_flushes = 0 + max_active_flushes = 0 + + def blocking_flush() -> None: + nonlocal active_flushes, max_active_flushes + with counter_lock: + active_flushes += 1 + max_active_flushes = max(max_active_flushes, active_flushes) + flush_entered.set() + release_flush.wait(timeout=1) + with counter_lock: + active_flushes -= 1 + + mock_client = MagicMock() + mock_client.flush.side_effect = blocking_flush + monkeypatch.setattr(_AppInsightsEventClient, "_initialized", True) + monkeypatch.setattr(_AppInsightsEventClient, "_client", mock_client) + _AppInsightsEventClient.track_event("test-event") + assert flush_entered.wait(timeout=1) + + shutdown_thread = threading.Thread(target=_AppInsightsEventClient._shutdown) + shutdown_thread.start() + shutdown_thread.join(timeout=0.05) + assert shutdown_thread.is_alive() + + release_flush.set() + shutdown_thread.join(timeout=1) + + assert not shutdown_thread.is_alive() + assert max_active_flushes == 1 + assert mock_client.flush.call_count == 2 + assert _AppInsightsEventClient._flush_thread is None + + def test_periodic_flush_does_not_block_event_tracking(self, monkeypatch): + """A slow flush must not block the event-producing thread.""" + FeatureFlags.configure_flags({PERIODIC_TELEMETRY_FLUSH_FEATURE_FLAG: True}) + flush_entered = threading.Event() + release_flush = threading.Event() + + def blocking_flush() -> None: + flush_entered.set() + release_flush.wait(timeout=1) + + mock_client = MagicMock() + mock_client.flush.side_effect = blocking_flush + monkeypatch.setattr(_AppInsightsEventClient, "_initialized", True) + monkeypatch.setattr(_AppInsightsEventClient, "_client", mock_client) + + flush_thread = threading.Thread(target=_AppInsightsEventClient.flush) + flush_thread.start() + assert flush_entered.wait(timeout=1) + + track_thread = threading.Thread( + target=_AppInsightsEventClient.track_event, + args=("event-during-flush",), + ) + track_thread.start() + track_thread.join(timeout=0.2) + + assert not track_thread.is_alive() + mock_client.track_event.assert_called_once() + + release_flush.set() + flush_thread.join(timeout=1) + assert not flush_thread.is_alive() + + def test_reset_prevents_worker_restart_during_shutdown(self, monkeypatch): + """Reset keeps worker lifecycle serialized until client state is cleared.""" + monkeypatch.setattr( + "uipath.telemetry._track._PERIODIC_TELEMETRY_FLUSH_INTERVAL_SECONDS", + 0.01, + ) + FeatureFlags.configure_flags({PERIODIC_TELEMETRY_FLUSH_FEATURE_FLAG: True}) + flush_entered = threading.Event() + release_flush = threading.Event() + + def blocking_flush() -> None: + flush_entered.set() + release_flush.wait(timeout=1) + + mock_client = MagicMock() + mock_client.flush.side_effect = blocking_flush + monkeypatch.setattr(_AppInsightsEventClient, "_initialized", True) + monkeypatch.setattr(_AppInsightsEventClient, "_client", mock_client) + _AppInsightsEventClient.track_event("start-worker") + assert flush_entered.wait(timeout=1) + + reset_thread = threading.Thread(target=_AppInsightsEventClient.reset) + reset_thread.start() + reset_thread.join(timeout=0.05) + assert reset_thread.is_alive() + + track_thread = threading.Thread( + target=_AppInsightsEventClient.track_event, + args=("event-during-reset",), + ) + track_thread.start() + track_thread.join(timeout=0.05) + assert track_thread.is_alive() + + release_flush.set() + reset_thread.join(timeout=1) + track_thread.join(timeout=1) + + assert not reset_thread.is_alive() + assert not track_thread.is_alive() + assert _AppInsightsEventClient._client is None + assert _AppInsightsEventClient._flush_thread is None + + def test_atexit_registration_uses_synchronized_shutdown(self, monkeypatch): + """The idempotent atexit callback owns worker shutdown and final flush.""" + monkeypatch.setattr(_AppInsightsEventClient, "_atexit_registered", False) + with patch("uipath.telemetry._track.atexit.register") as register: + _AppInsightsEventClient.register_atexit_flush() + _AppInsightsEventClient.register_atexit_flush() + + register.assert_called_once_with(_AppInsightsEventClient._shutdown) + class TestPublicProviderAndResetFunctions: """Test the public set_event_connection_string_provider and reset_event_client.""" diff --git a/packages/uipath/tests/tracing/test_otel_exporters.py b/packages/uipath/tests/tracing/test_otel_exporters.py index a55fa5d60..c8630b539 100644 --- a/packages/uipath/tests/tracing/test_otel_exporters.py +++ b/packages/uipath/tests/tracing/test_otel_exporters.py @@ -7,10 +7,12 @@ from opentelemetry.sdk.trace import ReadableSpan from opentelemetry.sdk.trace.export import SpanExportResult -from uipath.tracing._otel_exporters import ( - LlmOpsHttpExporter, - SpanStatus, +from uipath.platform.common._span_utils import SpanSource, SpanStatus +from uipath.platform.constants import ( + HEADER_INTERNAL_ACCOUNT_ID, + HEADER_INTERNAL_TENANT_ID, ) +from uipath.tracing._otel_exporters import LlmOpsHttpExporter @pytest.fixture @@ -54,7 +56,7 @@ def exporter(mock_env_vars): exporter = LlmOpsHttpExporter() # Mock _build_url to include query parameters as in the actual implementation exporter._build_url = MagicMock( # type: ignore - return_value="https://test.uipath.com/org/tenant/llmopstenant_/api/Traces/spans?traceId=test-trace-id&source=Robots" + return_value="https://test.uipath.com/org/tenant/llmopstenant_/api/Traces/v3/spans?traceId=test-trace-id&source=CodedAgents" ) yield exporter @@ -107,7 +109,7 @@ def test_export_success(exporter, mock_span): [{"span": "data", "TraceId": "test-trace-id"}] ) exporter.http_client.post.assert_called_once_with( - "https://test.uipath.com/org/tenant/llmopstenant_/api/Traces/spans?traceId=test-trace-id&source=Robots", + "https://test.uipath.com/org/tenant/llmopstenant_/api/Traces/v3/spans?traceId=test-trace-id&source=CodedAgents", json=[{"span": "data", "TraceId": "test-trace-id"}], ) @@ -159,6 +161,17 @@ def test_force_flush(exporter): assert exporter.force_flush() is True +def test_shutdown_closes_http_client(mock_env_vars): + with patch("uipath.tracing._otel_exporters.httpx.Client") as mock_client_class: + mock_client = MagicMock() + mock_client_class.return_value = mock_client + + exporter = LlmOpsHttpExporter() + exporter.shutdown() + + mock_client.close.assert_called_once_with() + + def test_get_base_url(): """Test _get_base_url method with different environment configurations.""" # Test with environment variable set @@ -232,11 +245,11 @@ def test_internal_headers_set_when_trace_base_url_present(): exporter = LlmOpsHttpExporter() assert ( - exporter.headers["X-UiPath-Internal-TenantId"] + exporter.headers[HEADER_INTERNAL_TENANT_ID] == "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" ) assert ( - exporter.headers["X-UiPath-Internal-AccountId"] + exporter.headers[HEADER_INTERNAL_ACCOUNT_ID] == "11111111-2222-3333-4444-555555555555" ) @@ -256,8 +269,8 @@ def test_internal_headers_not_set_without_trace_base_url(): with patch("uipath.tracing._otel_exporters.httpx.Client"): exporter = LlmOpsHttpExporter() - assert "X-UiPath-Internal-TenantId" not in exporter.headers - assert "X-UiPath-Internal-AccountId" not in exporter.headers + assert HEADER_INTERNAL_TENANT_ID not in exporter.headers + assert HEADER_INTERNAL_ACCOUNT_ID not in exporter.headers def test_send_with_retries_success(): @@ -277,6 +290,110 @@ def test_send_with_retries_success(): ) +def test_build_url_uses_v3_endpoint(mock_env_vars): + """_build_url must point to /api/Traces/v3/spans, not /api/Traces/spans.""" + with patch("uipath.tracing._otel_exporters.httpx.Client"): + exporter = LlmOpsHttpExporter() + span_list = [{"TraceId": "ab" * 16}] + url = exporter._build_url(span_list) + assert "/api/Traces/v3/spans" in url + # Ensure the v2 path (without /v3/) is not present + assert "/api/Traces/spans" not in url.replace("/api/Traces/v3/spans", "") + + +def test_build_url_uses_span_source_agents(mock_env_vars): + """_build_url must render the span's Source (Agents), not the hardcoded CodedAgents.""" + with patch("uipath.tracing._otel_exporters.httpx.Client"): + exporter = LlmOpsHttpExporter() + span_list = [{"TraceId": "ab" * 16, "Source": SpanSource.AGENTS}] + url = exporter._build_url(span_list) + assert "&source=Agents" in url + assert "&source=CodedAgents" not in url + assert "/api/Traces/v3/spans" in url + + +def test_build_url_uses_span_source_coded_agents(mock_env_vars): + """An explicit CodedAgents Source still renders CodedAgents.""" + with patch("uipath.tracing._otel_exporters.httpx.Client"): + exporter = LlmOpsHttpExporter() + span_list = [{"TraceId": "ab" * 16, "Source": SpanSource.CODED_AGENTS}] + url = exporter._build_url(span_list) + assert "&source=CodedAgents" in url + + +def test_build_url_defaults_to_coded_agents_when_source_missing(mock_env_vars): + """When the span dict has no Source key, default to CodedAgents (back-compat).""" + with patch("uipath.tracing._otel_exporters.httpx.Client"): + exporter = LlmOpsHttpExporter() + span_list = [{"TraceId": "ab" * 16}] + url = exporter._build_url(span_list) + assert "&source=CodedAgents" in url + + +def test_agent_builder_span_yields_source_agents(mock_env_vars): + """A span with uipath.source=1 must flow through to &source=Agents in the URL. + + Drives a real span dict through otel_span_to_uipath_span().to_dict() rather + than hand-building it, guarding the whole attribute->Source->URL path. + """ + from opentelemetry.trace import SpanContext, StatusCode + + from uipath.platform.common import _SpanUtils + + # otel_span_to_uipath_span reads the context via get_span_context() and + # formats trace_id/span_id as hex, so provide a real SpanContext. + span = MagicMock(spec=ReadableSpan) + span.get_span_context.return_value = SpanContext( + trace_id=0xABCDEF1234567890ABCDEF1234567890, + span_id=0x1234567890ABCDEF, + is_remote=False, + ) + span.parent = None + span.name = "agent-span" + span.status.status_code = StatusCode.OK + span.status.description = None + span.attributes = { + "uipath.custom_instrumentation": True, + "uipath.source": 1, # SourceEnum.Agents + } + span.events = [] + span.links = [] + span.start_time = 0 + span.end_time = 1 + + span_dict = _SpanUtils.otel_span_to_uipath_span( + span, serialize_attributes=False + ).to_dict(serialize_attributes=False) + assert str(span_dict["Source"]) == "Agents" + + with patch("uipath.tracing._otel_exporters.httpx.Client"): + exporter = LlmOpsHttpExporter() + url = exporter._build_url([span_dict]) + assert "&source=Agents" in url + + +def test_determine_status_ok_returns_string(mock_env_vars): + with patch("uipath.tracing._otel_exporters.httpx.Client"): + exporter = LlmOpsHttpExporter() + assert exporter._determine_status(None) == "Ok" + assert exporter._determine_status(None) == SpanStatus.OK + + +def test_determine_status_error_returns_string(mock_env_vars): + with patch("uipath.tracing._otel_exporters.httpx.Client"): + exporter = LlmOpsHttpExporter() + assert exporter._determine_status("some error") == "Error" + assert exporter._determine_status("some error") == SpanStatus.ERROR + + +def test_determine_status_graph_interrupt_returns_running(mock_env_vars): + with patch("uipath.tracing._otel_exporters.httpx.Client"): + exporter = LlmOpsHttpExporter() + # GraphInterrupt is a HITL pause (still in-progress), not a terminal abort. + assert exporter._determine_status("GraphInterrupt()") == "Running" + assert exporter._determine_status("GraphInterrupt()") == SpanStatus.RUNNING + + class TestLangchainExporter(unittest.TestCase): def setUp(self): self.exporter = LlmOpsHttpExporter() @@ -685,7 +802,7 @@ def exporter_with_mocks(self, mock_env_vars): with patch("uipath.tracing._otel_exporters.httpx.Client"): exporter = LlmOpsHttpExporter() exporter._build_url = MagicMock( # type: ignore - return_value="https://test.uipath.com/org/tenant/llmopstenant_/api/Traces/spans?traceId=test-trace-id&source=Robots" + return_value="https://test.uipath.com/org/tenant/llmopstenant_/api/Traces/v3/spans?traceId=test-trace-id&source=CodedAgents" ) yield exporter @@ -810,5 +927,99 @@ def test_none_stays_none(self, mock_env_vars, mock_span): assert payload["ProcessKey"] is None +class TestVerbosityLevelReexport: + """VerbosityLevel from uipath-platform is re-exported via uipath.tracing.""" + + def test_uipath_tracing_reexports_verbosity_level(self) -> None: + from uipath.platform.common._span_utils import ( + VerbosityLevel as _CommonVerbosity, + ) + from uipath.tracing import VerbosityLevel as _TracingVerbosity + + assert _TracingVerbosity is _CommonVerbosity + assert _TracingVerbosity.OFF == "Off" + + +class TestV3EndToEnd: + """Integration-style tests verifying string enum values reach the v3 URL end-to-end.""" + + def _make_real_otel_span(self, status_code=None): + """Build a minimal mock OTel ReadableSpan with a real SpanContext.""" + from datetime import datetime + from unittest.mock import Mock + + from opentelemetry.trace import SpanContext, StatusCode + + if status_code is None: + status_code = StatusCode.OK + + mock_span = Mock(spec=ReadableSpan) + mock_context = SpanContext( + trace_id=0xABCDEF1234567890ABCDEF1234567890, + span_id=0x1234567890ABCDEF, + is_remote=False, + ) + mock_span.get_span_context.return_value = mock_context + mock_span.name = "test-v3-span" + mock_span.parent = None + mock_span.status.status_code = status_code + mock_span.status.description = None + mock_span.attributes = {"uipath.custom_instrumentation": True} + mock_span.events = [] + mock_span.links = [] + now_ns = int(datetime.now().timestamp() * 1e9) + mock_span.start_time = now_ns + mock_span.end_time = now_ns + 1_000_000 + return mock_span + + def test_export_posts_to_v3_url_with_string_enums(self, mock_env_vars): + """Exporting a span must POST to /api/Traces/v3/spans with string Status and Source.""" + from opentelemetry.trace import StatusCode + + otel_span = self._make_real_otel_span(status_code=StatusCode.OK) + + with patch("uipath.tracing._otel_exporters.httpx.Client") as mock_client_cls: + mock_client = MagicMock() + mock_client_cls.return_value = mock_client + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_client.post.return_value = mock_response + + exporter = LlmOpsHttpExporter() + result = exporter.export([otel_span]) + + assert result == SpanExportResult.SUCCESS + + # Verify the POST was made + mock_client.post.assert_called_once() + call_args = mock_client.post.call_args + + # URL must contain v3/spans + posted_url = ( + call_args.args[0] if call_args.args else call_args.kwargs.get("url", "") + ) + assert "v3/spans" in posted_url, f"Expected v3/spans in URL, got: {posted_url}" + + # Body must contain string enum values, not integers + payload: list[dict[str, object]] = ( + call_args.kwargs.get("json") or call_args.args[1] + ) + assert len(payload) == 1 + span_payload = payload[0] + + # Status should be the string "Ok", not integer 1 + assert span_payload["Status"] == "Ok", ( + f"Expected Status='Ok' (string), got {span_payload['Status']!r}" + ) + assert span_payload["Status"] != 1, "Status must not be integer 1 (v2 format)" + + # Source should be the string "CodedAgents", not integer 10 + assert span_payload["Source"] == "CodedAgents", ( + f"Expected Source='CodedAgents' (string), got {span_payload['Source']!r}" + ) + assert span_payload["Source"] != 10, "Source must not be integer 10 (v2 format)" + + if __name__ == "__main__": unittest.main() diff --git a/packages/uipath/uv.lock b/packages/uipath/uv.lock index 77434aaa8..c905ec7c3 100644 --- a/packages/uipath/uv.lock +++ b/packages/uipath/uv.lock @@ -2,6 +2,16 @@ version = 1 revision = 3 requires-python = ">=3.11" +[options] +exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. +exclude-newer-span = "P2D" + +[options.exclude-newer-package] +uipath-ipc = false +uipath-runtime = false +uipath-platform = false +uipath-core = false + [[package]] name = "aiohappyeyeballs" version = "2.6.1" @@ -342,6 +352,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445, upload-time = "2025-11-19T20:55:50.744Z" }, ] +[[package]] +name = "chardet" +version = "7.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/b6/9df434a8eeba2e6628c465a1dfa31034228ef79b26f76f46278f4ef7e49d/chardet-7.4.3.tar.gz", hash = "sha256:cc1d4eb92a4ec1c2df3b490836ffa46922e599d34ce0bb75cf41fd2bf6303d56", size = 784800, upload-time = "2026-04-13T21:33:39.803Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/52/505c207f334d51e937cbaa27ff95776e16e2d120e13cbe491cd7b3a70b50/chardet-7.4.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:25a862cddc6a9ac07023e808aedd297115345fbaabc2690479481ddc0f980e09", size = 870747, upload-time = "2026-04-13T21:32:56.916Z" }, + { url = "https://files.pythonhosted.org/packages/14/4b/d3c79495dee4831b8bebca2790e72cb90f0c5849c940570a7c7e5b70b952/chardet-7.4.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7005c88da26fd95d8abb8acbe6281d833e9a9181b03cf49b4546c4555389bd97", size = 853210, upload-time = "2026-04-13T21:32:58.309Z" }, + { url = "https://files.pythonhosted.org/packages/b9/99/f6a822ad1bde25a4c38dc3e770485e78e0893dfd871cd6e18ed3ea3a795e/chardet-7.4.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc50f28bad067393cce0af9091052c3b8df7a23115afd8ba7b2e0947f0cef1f8", size = 873625, upload-time = "2026-04-13T21:32:59.606Z" }, + { url = "https://files.pythonhosted.org/packages/b1/10/31932775c94a86814f76b41c4a772b52abfb0e6125324f32c6da1196c297/chardet-7.4.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4c3da294de1a681097848ab58bd3f2771a674f8039d2d87a5538b28856b815e9", size = 883436, upload-time = "2026-04-13T21:33:01.351Z" }, + { url = "https://files.pythonhosted.org/packages/6c/63/0f43e3acf2c436fdb32a0f904aeb03a2904d2126eed34a042a194d235926/chardet-7.4.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:93c45e116dd51b66226a53ade3f9f635e870de5399b90e00ce45dcc311093bf4", size = 876589, upload-time = "2026-04-13T21:33:02.636Z" }, + { url = "https://files.pythonhosted.org/packages/5d/a6/e9b8f8a3e99602792b01fa7d0a731737615ab56d8bfd0b52935a0ef88b85/chardet-7.4.3-cp311-cp311-win_amd64.whl", hash = "sha256:ccc1f83ab4bcfb901cf39e0c4ba6bc6e726fc6264735f10e24ceb5cb47387578", size = 941866, upload-time = "2026-04-13T21:33:04.282Z" }, + { url = "https://files.pythonhosted.org/packages/61/33/29de185079e6675c3f375546e30a559b7ddc75ce972f18d6e566cd9ea4eb/chardet-7.4.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:75d3c65cc16bddf40b8da1fd25ba84fca5f8070f2b14e86083653c1c85aee971", size = 874870, upload-time = "2026-04-13T21:33:05.977Z" }, + { url = "https://files.pythonhosted.org/packages/9c/2f/4c5af01fd1a7506a1d5375403d68925eac70289229492db5aa68b58103d8/chardet-7.4.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:29af5999f654e8729d251f1724a62b538b1262d9292cccaefddf8a02aae1ef6a", size = 854859, upload-time = "2026-04-13T21:33:07.381Z" }, + { url = "https://files.pythonhosted.org/packages/36/21/edb36ad5dfa48d7f8eed97ab43931ecdaa8c15166c21b1d614967e49d681/chardet-7.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:626f00299ad62dfe937058a09572beed442ccc7b58f87aa667949b20fd3db235", size = 875032, upload-time = "2026-04-13T21:33:08.741Z" }, + { url = "https://files.pythonhosted.org/packages/e5/59/a32a241d861cf180853a11c8e5a67641cb1b2af13c3a5ccce83ec07e2c9f/chardet-7.4.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9a4904dd5f071b7a7d7f50b4a67a86db3c902d243bf31708f1d5cde2f68239cb", size = 888283, upload-time = "2026-04-13T21:33:10.213Z" }, + { url = "https://files.pythonhosted.org/packages/87/2e/e1ee6a77abf3782c00e05b89c4d4328c8353bf9500661c4348df1dd68614/chardet-7.4.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d2879598bc220689e8ce509fe9c3f37ad2fca53a36be9c9bd91abdd91dd364f", size = 879974, upload-time = "2026-04-13T21:33:11.448Z" }, + { url = "https://files.pythonhosted.org/packages/32/60/fca69c534602a7ced04280c952a246ad1edde2a6ca3a164f65d32ac41fe7/chardet-7.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:4b2799bd58e7245cfa8d4ab2e8ad1d76a5c3a5b1f32318eb6acca4c69a3e7101", size = 943973, upload-time = "2026-04-13T21:33:12.756Z" }, + { url = "https://files.pythonhosted.org/packages/7c/43/79ac9b4db5bc87020c9dbc419125371d80882d1d197e9c4765ba8682b605/chardet-7.4.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a9e4486df251b8962e86ea9f139ca235aa6e0542a00f7844c9a04160afb99aa9", size = 873769, upload-time = "2026-04-13T21:33:14.002Z" }, + { url = "https://files.pythonhosted.org/packages/55/5f/25bdec773905bff0ff6cf35ca73b17bd05593b4f87bd8c5fa43705f7167d/chardet-7.4.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4fbff1907925b0c5a1064cffb5e040cd5e338585c9c552625f30de6bc2f3107a", size = 853991, upload-time = "2026-04-13T21:33:15.564Z" }, + { url = "https://files.pythonhosted.org/packages/b4/07/a29380ee0b215d23d77733b5ad60c5c0c7969650e080c667acdf9462040d/chardet-7.4.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:365135eaf37ba65a828f8e668eb0a8c38c479dcbec724dc25f4dfd781049c357", size = 874024, upload-time = "2026-04-13T21:33:16.915Z" }, + { url = "https://files.pythonhosted.org/packages/a8/b1/3338e121cbd4c8a126b8ccb1061170c2ce51a53f678c502793ea49c6fd6d/chardet-7.4.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfc134b70c846c21ead8e43ada3ae1a805fff732f6922f8abcf2ff27b8f6493d", size = 887410, upload-time = "2026-04-13T21:33:18.368Z" }, + { url = "https://files.pythonhosted.org/packages/63/1c/44a9a9e0c59c185a5d307ceaeee8768afa1558f0a24f7a4b5fa11b67586b/chardet-7.4.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9acd9988a93e09390f3cd231201ea7166c415eb8da1b735928990ffc05cb9fbb", size = 879269, upload-time = "2026-04-13T21:33:20.377Z" }, + { url = "https://files.pythonhosted.org/packages/1b/b3/5d0e77ea774bd3224321c248880ea0c0379000ac5c2bb6d77609549de247/chardet-7.4.3-cp313-cp313-win_amd64.whl", hash = "sha256:e1b98790c284ff813f18f7cf7de5f05ea2435a080030c7f1a8318f3a4f80b131", size = 944155, upload-time = "2026-04-13T21:33:21.694Z" }, + { url = "https://files.pythonhosted.org/packages/70/a8/bf0811d859e13801279a2ae64f37a408027b282f2047bc0001c75dd356ad/chardet-7.4.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d892d3dcd652fdef53e3d6327d39b17c0df40a899dfc919abaeb64c974497531", size = 872887, upload-time = "2026-04-13T21:33:23.328Z" }, + { url = "https://files.pythonhosted.org/packages/51/ac/b9d68ebddfe1b02c77af5bf81120e12b036b4432dc6af7a303d90e2bc38b/chardet-7.4.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:acc46d1b8b7d5783216afe15db56d1c179b9a40e5a1558bc13164c4fd20674c4", size = 853964, upload-time = "2026-04-13T21:33:24.724Z" }, + { url = "https://files.pythonhosted.org/packages/2a/81/17fa103ea9caf5d325a5e4051ab2ba65996fd66baa60b81ee41af1f54e10/chardet-7.4.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ac3bf11c645734a1701a3804e43eabd98851838192267d08c353a834ab79fea", size = 876006, upload-time = "2026-04-13T21:33:26.098Z" }, + { url = "https://files.pythonhosted.org/packages/c2/20/193faab46a68ea550587331a698c3dca8099f8901d10937c4443135c7ed9/chardet-7.4.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e3bd9f936e04bae89c254262af08d9e5b98f805175ba1e29d454e6cba3107b7", size = 887680, upload-time = "2026-04-13T21:33:27.49Z" }, + { url = "https://files.pythonhosted.org/packages/40/c6/94a3c673327392652ee8bdea9a45bc8a5f5365197a7387d68f0eed007115/chardet-7.4.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:27cc23da03630cdecc9aa81a895aa86629c211f995cd57651f0fbc280717bf93", size = 879865, upload-time = "2026-04-13T21:33:29.052Z" }, + { url = "https://files.pythonhosted.org/packages/b1/2c/cad8b5e3623a987f3c930b68e2bdd06cfc388cd91cd42ed05f1227701b73/chardet-7.4.3-cp314-cp314-win_amd64.whl", hash = "sha256:b95c934b9ad59e2ba8abb9be49df70d3ad1b0d95d864b9fdb7588d4fa8bd921c", size = 939594, upload-time = "2026-04-13T21:33:31.391Z" }, + { url = "https://files.pythonhosted.org/packages/33/e0/d06e42fd6f02a58e5e227e5106587751cb38adcff0aaf949add744b78b6e/chardet-7.4.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c77867f0c1cb8bd819502249fcdc500364aedb07881e11b743726fa2148e7b6e", size = 889714, upload-time = "2026-04-13T21:33:32.772Z" }, + { url = "https://files.pythonhosted.org/packages/d4/ed/40d091954d48abea037baae6be8fb79905e5f78d34d12ea955132c7d8011/chardet-7.4.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:cf1efeaf65a6ef2f5b9cc3a1df6f08ba2831b369ccaa4c7018eaf90aa757bb11", size = 872319, upload-time = "2026-04-13T21:33:34.427Z" }, + { url = "https://files.pythonhosted.org/packages/bb/77/82a46821dbfbdfe062710d2bf2ede13426304e3567a23c57d919c0c31630/chardet-7.4.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9f3504c139a2ad544077dd2d9e412cd08b01786843d76997cd43bb6de311723c", size = 892021, upload-time = "2026-04-13T21:33:35.766Z" }, + { url = "https://files.pythonhosted.org/packages/49/57/42d30c562bda5b4a839766c1aad8d5856b798ad2a1c3247b72a679afec94/chardet-7.4.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457f619882ba66327d4d8d14c6c342269bdb1e4e1c38e8117df941d14d351b04", size = 902509, upload-time = "2026-04-13T21:33:37.096Z" }, + { url = "https://files.pythonhosted.org/packages/8c/6c/0a40afdb50a0fe041ab95553b835a8160b6cf0e81edf2ae2fe9f5224cbf9/chardet-7.4.3-py3-none-any.whl", hash = "sha256:1173b74051570cf08099d7429d92e4882d375ad4217f92a6e5240ccfb26f231e", size = 626562, upload-time = "2026-04-13T21:33:38.559Z" }, +] + [[package]] name = "charset-normalizer" version = "3.4.4" @@ -2511,6 +2558,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, ] +[[package]] +name = "types-pyyaml" +version = "6.0.12.20260518" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b8/83/4a1afc3fbfcf5b8d46fc390cd95ed6b0dc9010a265f4e9f46314efffa37a/types_pyyaml-6.0.12.20260518.tar.gz", hash = "sha256:d917f83fb38462550338c1297faedd860b3ec83912b96b1e3d73255f7473e466", size = 17850, upload-time = "2026-05-18T06:01:58.675Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/a2/c01db32be2ae7d6a1689972f3c492b149ee4e164b12fdfd9f64b50888215/types_pyyaml-6.0.12.20260518-py3-none-any.whl", hash = "sha256:d2150f75a231c9fe9c7463bd29487d93e60bac90400287351384bc2284eba7cd", size = 20312, upload-time = "2026-05-18T06:01:57.368Z" }, +] + [[package]] name = "types-toml" version = "0.10.8.20240310" @@ -2543,7 +2599,7 @@ wheels = [ [[package]] name = "uipath" -version = "2.10.40" +version = "2.14.1" source = { editable = "." } dependencies = [ { name = "applicationinsights" }, @@ -2559,6 +2615,7 @@ dependencies = [ { name = "pysignalr" }, { name = "python-dotenv" }, { name = "python-socketio" }, + { name = "pyyaml" }, { name = "rich" }, { name = "tenacity" }, { name = "truststore" }, @@ -2567,6 +2624,11 @@ dependencies = [ { name = "uipath-runtime" }, ] +[package.optional-dependencies] +ipc = [ + { name = "uipath-ipc" }, +] + [package.dev-dependencies] dev = [ { name = "bandit" }, @@ -2592,7 +2654,9 @@ dev = [ { name = "rust-just" }, { name = "termynal" }, { name = "tomli-w" }, + { name = "types-pyyaml" }, { name = "types-toml" }, + { name = "uipath-ipc" }, { name = "virtualenv" }, ] @@ -2611,13 +2675,16 @@ requires-dist = [ { name = "pysignalr", specifier = "==1.3.0" }, { name = "python-dotenv", specifier = ">=1.0.1" }, { name = "python-socketio", specifier = ">=5.15.0,<6.0.0" }, + { name = "pyyaml", specifier = ">=6.0,<7.0" }, { name = "rich", specifier = ">=14.2.0" }, { name = "tenacity", specifier = ">=9.0.0" }, { name = "truststore", specifier = ">=0.10.1" }, { name = "uipath-core", editable = "../uipath-core" }, + { name = "uipath-ipc", marker = "extra == 'ipc'", specifier = ">=2.5.1,<2.6.0" }, { name = "uipath-platform", editable = "../uipath-platform" }, - { name = "uipath-runtime", specifier = ">=0.10.0,<0.11.0" }, + { name = "uipath-runtime", specifier = ">=0.13.0,<0.14.0" }, ] +provides-extras = ["ipc"] [package.metadata.requires-dev] dev = [ @@ -2644,13 +2711,15 @@ dev = [ { name = "rust-just", specifier = ">=1.39.0" }, { name = "termynal", specifier = ">=0.13.1" }, { name = "tomli-w", specifier = ">=1.2.0" }, + { name = "types-pyyaml", specifier = ">=6.0" }, { name = "types-toml", specifier = ">=0.10.8" }, + { name = "uipath-ipc", specifier = ">=2.5.1,<2.6.0" }, { name = "virtualenv", specifier = ">=20.36.1" }, ] [[package]] name = "uipath-core" -version = "0.5.10" +version = "0.5.31" source = { editable = "../uipath-core" } dependencies = [ { name = "opentelemetry-instrumentation" }, @@ -2680,11 +2749,21 @@ dev = [ { name = "rust-just", specifier = ">=1.39.0" }, ] +[[package]] +name = "uipath-ipc" +version = "2.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/6b/53d9725d6abd1dab447300a7f999332f530678e3fabd38fc31171a5a9a6f/uipath_ipc-2.5.2.tar.gz", hash = "sha256:d69c3d7c1ad1a25ef7f9f1d78c505f5d2ed7daa7227bb202404fa3d47e0ba12e", size = 86360, upload-time = "2026-07-24T09:44:35.159Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/eb/6def505a0d351119da27b342c11eb0767a31a7f100022d35e349c5194eb3/uipath_ipc-2.5.2-py3-none-any.whl", hash = "sha256:617f25f35377d87956165a875b0966a3cd69ae6724fcd533c93a7a6a9460fc4f", size = 50345, upload-time = "2026-07-24T09:44:33.7Z" }, +] + [[package]] name = "uipath-platform" -version = "0.1.18" +version = "0.2.17" source = { editable = "../uipath-platform" } dependencies = [ + { name = "anyio" }, { name = "httpx" }, { name = "pydantic-function-models" }, { name = "sqlparse" }, @@ -2695,6 +2774,7 @@ dependencies = [ [package.metadata] requires-dist = [ + { name = "anyio", specifier = ">=4.0.0" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "pydantic-function-models", specifier = ">=0.1.11" }, { name = "sqlparse", specifier = ">=0.5.5" }, @@ -2720,14 +2800,16 @@ dev = [ [[package]] name = "uipath-runtime" -version = "0.10.0" +version = "0.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "chardet" }, { name = "uipath-core" }, + { name = "vadersentiment" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/75/64/69462ee01a5607ce36b1fa152c52ac72fb28abe0aa049394406fc0b31525/uipath_runtime-0.10.0.tar.gz", hash = "sha256:d27d58e2252f506c8c0e00f814b37c3863150e8ffcde8e4c6ab14bd98febd3df", size = 139626, upload-time = "2026-03-24T19:42:43.738Z" } +sdist = { url = "https://files.pythonhosted.org/packages/65/c0/d55cf48ee43758c2c1c65f52fd0596fd75f5bd5067a5016e6553b4d2c5fe/uipath_runtime-0.13.0.tar.gz", hash = "sha256:8ae3150df5fb0043210faacf39148789c1fb252c6a8d6bc27174c0d9ce7304f5", size = 243594, upload-time = "2026-08-04T11:19:04.886Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/ed/9c0e97a078b96e4d3742ea3515cb30886b08579cd08077cd42a159adf70d/uipath_runtime-0.10.0-py3-none-any.whl", hash = "sha256:4f52df0b56f54e70fcf34fbf74e223d02b97b5a6fd6d8f64bc06782bb5484b07", size = 42097, upload-time = "2026-03-24T19:42:42.359Z" }, + { url = "https://files.pythonhosted.org/packages/dc/88/9518dcbeedaa8117e5fd3d0424c4a7354bae1f3ede1b2e3f48524e4b7efc/uipath_runtime-0.13.0-py3-none-any.whl", hash = "sha256:2a7c128cf14fdbef899b272ee5995da63baeb882acc904f39ef8f8f52bcb019f", size = 96349, upload-time = "2026-08-04T11:19:03.376Z" }, ] [[package]] @@ -2739,6 +2821,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, ] +[[package]] +name = "vadersentiment" +version = "3.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/77/8c/4a48c10a50f750ae565e341e697d74a38075a3e43ff0df6f1ab72e186902/vaderSentiment-3.3.2.tar.gz", hash = "sha256:5d7c06e027fc8b99238edb0d53d970cf97066ef97654009890b83703849632f9", size = 2466783, upload-time = "2020-05-22T15:06:32.81Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/fc/310e16254683c1ed35eeb97386986d6c00bc29df17ce280aed64d55537e9/vaderSentiment-3.3.2-py2.py3-none-any.whl", hash = "sha256:3bf1d243b98b1afad575b9f22bc2cb1e212b94ff89ca74f8a23a588d024ea311", size = 125950, upload-time = "2020-05-22T15:07:00.052Z" }, +] + [[package]] name = "virtualenv" version = "20.36.1" diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 000000000..5c593c8ba --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,27 @@ +sonar.projectKey=UiPath_uipath-python +sonar.organization=ui +sonar.host.url=https://sonarcloud.io + +sonar.sources=packages/uipath/src,packages/uipath-core/src,packages/uipath-platform/src +sonar.tests=packages/uipath/tests,packages/uipath-core/tests,packages/uipath-platform/tests + +sonar.python.version=3.11,3.12,3.13 +sonar.python.coverage.reportPaths=packages/uipath/coverage.xml,packages/uipath-core/coverage.xml,packages/uipath-platform/coverage.xml + +sonar.exclusions=**/samples/**,**/testcases/**,**/template/**,**/_resources/** + +sonar.cpd.exclusions=**/__init__.py + +# The uipath-ipc runtime contract (methods + DTO fields) is PascalCase because +# the wire keys and method names are dictated by the .NET/CoreIpc peer — the +# serializer maps them verbatim (no alias mechanism), so the names cannot be +# Python-idiomatic without breaking interop. It is isolated in cli_server_ipc.py +# so this suppression of method-naming (S100) and field-naming (S116) touches +# that file only. +sonar.issue.ignore.multicriteria=ipc1,ipc2 +sonar.issue.ignore.multicriteria.ipc1.ruleKey=python:S100 +sonar.issue.ignore.multicriteria.ipc1.resourceKey=**/cli_server_ipc.py +sonar.issue.ignore.multicriteria.ipc2.ruleKey=python:S116 +sonar.issue.ignore.multicriteria.ipc2.resourceKey=**/cli_server_ipc.py + +sonar.sourceEncoding=UTF-8