chore: initial public snapshot for github upload
This commit is contained in:
Your Name
@@ -0,0 +1,3 @@
|
||||
from .deepeval import DeepEvalLogger
|
||||
|
||||
__all__ = ["DeepEvalLogger"]
|
||||
@@ -0,0 +1,120 @@
|
||||
# duplicate -> https://github.com/confident-ai/deepeval/blob/main/deepeval/confident/api.py
|
||||
import logging
|
||||
import httpx
|
||||
from enum import Enum
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
DEEPEVAL_BASE_URL = "https://deepeval.confident-ai.com"
|
||||
DEEPEVAL_BASE_URL_EU = "https://eu.deepeval.confident-ai.com"
|
||||
API_BASE_URL = "https://api.confident-ai.com"
|
||||
API_BASE_URL_EU = "https://eu.api.confident-ai.com"
|
||||
retryable_exceptions = httpx.HTTPError
|
||||
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
HTTPHandler,
|
||||
get_async_httpx_client,
|
||||
httpxSpecialProvider,
|
||||
)
|
||||
|
||||
|
||||
def log_retry_error(details):
|
||||
exception = details.get("exception")
|
||||
tries = details.get("tries")
|
||||
if exception:
|
||||
logging.error(f"Confident AI Error: {exception}. Retrying: {tries} time(s)...")
|
||||
else:
|
||||
logging.error(f"Retrying: {tries} time(s)...")
|
||||
|
||||
|
||||
class HttpMethods(Enum):
|
||||
GET = "GET"
|
||||
POST = "POST"
|
||||
DELETE = "DELETE"
|
||||
PUT = "PUT"
|
||||
|
||||
|
||||
class Endpoints(Enum):
|
||||
DATASET_ENDPOINT = "/v1/dataset"
|
||||
TEST_RUN_ENDPOINT = "/v1/test-run"
|
||||
TRACING_ENDPOINT = "/v1/tracing"
|
||||
EVENT_ENDPOINT = "/v1/event"
|
||||
FEEDBACK_ENDPOINT = "/v1/feedback"
|
||||
PROMPT_ENDPOINT = "/v1/prompt"
|
||||
RECOMMEND_ENDPOINT = "/v1/recommend-metrics"
|
||||
EVALUATE_ENDPOINT = "/evaluate"
|
||||
GUARD_ENDPOINT = "/guard"
|
||||
GUARDRAILS_ENDPOINT = "/guardrails"
|
||||
BASELINE_ATTACKS_ENDPOINT = "/generate-baseline-attacks"
|
||||
|
||||
|
||||
class Api:
|
||||
def __init__(self, api_key: str, base_url=None):
|
||||
self.api_key = api_key
|
||||
self._headers = {
|
||||
"Content-Type": "application/json",
|
||||
# "User-Agent": "Python/Requests",
|
||||
"CONFIDENT_API_KEY": api_key,
|
||||
}
|
||||
# using the global non-eu variable for base url
|
||||
self.base_api_url = base_url or API_BASE_URL
|
||||
self.sync_http_handler = HTTPHandler()
|
||||
self.async_http_handler = get_async_httpx_client(
|
||||
llm_provider=httpxSpecialProvider.LoggingCallback
|
||||
)
|
||||
|
||||
def _http_request(
|
||||
self, method: str, url: str, headers=None, json=None, params=None
|
||||
):
|
||||
if method != "POST":
|
||||
raise Exception("Only POST requests are supported")
|
||||
try:
|
||||
self.sync_http_handler.post(
|
||||
url=url,
|
||||
headers=headers,
|
||||
json=json,
|
||||
params=params,
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise Exception(f"DeepEval logging error: {e.response.text}")
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
def send_request(
|
||||
self, method: HttpMethods, endpoint: Endpoints, body=None, params=None
|
||||
):
|
||||
url = f"{self.base_api_url}{endpoint.value}"
|
||||
res = self._http_request(
|
||||
method=method.value,
|
||||
url=url,
|
||||
headers=self._headers,
|
||||
json=body,
|
||||
params=params,
|
||||
)
|
||||
|
||||
if res.status_code == 200:
|
||||
try:
|
||||
return res.json()
|
||||
except ValueError:
|
||||
return res.text
|
||||
else:
|
||||
verbose_logger.debug(res.json())
|
||||
raise Exception(res.json().get("error", res.text))
|
||||
|
||||
async def a_send_request(
|
||||
self, method: HttpMethods, endpoint: Endpoints, body=None, params=None
|
||||
):
|
||||
if method != HttpMethods.POST:
|
||||
raise Exception("Only POST requests are supported")
|
||||
|
||||
url = f"{self.base_api_url}{endpoint.value}"
|
||||
try:
|
||||
await self.async_http_handler.post(
|
||||
url=url,
|
||||
headers=self._headers,
|
||||
json=body,
|
||||
params=params,
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise Exception(f"DeepEval logging error: {e.response.text}")
|
||||
except Exception as e:
|
||||
raise e
|
||||
@@ -0,0 +1,175 @@
|
||||
import os
|
||||
from litellm._uuid import uuid
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.deepeval.api import Api, Endpoints, HttpMethods
|
||||
from litellm.integrations.deepeval.types import (
|
||||
BaseApiSpan,
|
||||
SpanApiType,
|
||||
TraceApi,
|
||||
TraceSpanApiStatus,
|
||||
)
|
||||
from litellm.integrations.deepeval.utils import (
|
||||
to_zod_compatible_iso,
|
||||
validate_environment,
|
||||
)
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
|
||||
# This file includes the custom callbacks for LiteLLM Proxy
|
||||
# Once defined, these can be passed in proxy_config.yaml
|
||||
class DeepEvalLogger(CustomLogger):
|
||||
"""Logs litellm traces to DeepEval's platform."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
api_key = os.getenv("CONFIDENT_API_KEY")
|
||||
self.litellm_environment = os.getenv("LITELM_ENVIRONMENT", "development")
|
||||
validate_environment(self.litellm_environment)
|
||||
if not api_key:
|
||||
raise ValueError(
|
||||
"Please set 'CONFIDENT_API_KEY=<>' in your environment variables."
|
||||
)
|
||||
self.api = Api(api_key=api_key)
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
"""Logs a success event to DeepEval's platform."""
|
||||
self._sync_event_handler(
|
||||
kwargs, response_obj, start_time, end_time, is_success=True
|
||||
)
|
||||
|
||||
def log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
"""Logs a failure event to DeepEval's platform."""
|
||||
self._sync_event_handler(
|
||||
kwargs, response_obj, start_time, end_time, is_success=False
|
||||
)
|
||||
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
"""Logs a failure event to DeepEval's platform."""
|
||||
await self._async_event_handler(
|
||||
kwargs, response_obj, start_time, end_time, is_success=False
|
||||
)
|
||||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
"""Logs a success event to DeepEval's platform."""
|
||||
await self._async_event_handler(
|
||||
kwargs, response_obj, start_time, end_time, is_success=True
|
||||
)
|
||||
|
||||
def _prepare_trace_api(
|
||||
self, kwargs, response_obj, start_time, end_time, is_success
|
||||
):
|
||||
_start_time = to_zod_compatible_iso(start_time)
|
||||
_end_time = to_zod_compatible_iso(end_time)
|
||||
_standard_logging_object = kwargs.get("standard_logging_object", {})
|
||||
base_api_span = self._create_base_api_span(
|
||||
kwargs,
|
||||
standard_logging_object=_standard_logging_object,
|
||||
start_time=_start_time,
|
||||
end_time=_end_time,
|
||||
is_success=is_success,
|
||||
)
|
||||
trace_api = self._create_trace_api(
|
||||
base_api_span,
|
||||
standard_logging_object=_standard_logging_object,
|
||||
start_time=_start_time,
|
||||
end_time=_end_time,
|
||||
litellm_environment=self.litellm_environment,
|
||||
)
|
||||
|
||||
body = {}
|
||||
|
||||
try:
|
||||
body = trace_api.model_dump(by_alias=True, exclude_none=True)
|
||||
except AttributeError:
|
||||
# Pydantic version below 2.0
|
||||
body = trace_api.dict(by_alias=True, exclude_none=True)
|
||||
return body
|
||||
|
||||
def _sync_event_handler(
|
||||
self, kwargs, response_obj, start_time, end_time, is_success
|
||||
):
|
||||
body = self._prepare_trace_api(
|
||||
kwargs, response_obj, start_time, end_time, is_success
|
||||
)
|
||||
try:
|
||||
response = self.api.send_request(
|
||||
method=HttpMethods.POST,
|
||||
endpoint=Endpoints.TRACING_ENDPOINT,
|
||||
body=body,
|
||||
)
|
||||
except Exception as e:
|
||||
raise e
|
||||
verbose_logger.debug(
|
||||
"DeepEvalLogger: sync_log_failure_event: Api response %s", response
|
||||
)
|
||||
|
||||
async def _async_event_handler(
|
||||
self, kwargs, response_obj, start_time, end_time, is_success
|
||||
):
|
||||
body = self._prepare_trace_api(
|
||||
kwargs, response_obj, start_time, end_time, is_success
|
||||
)
|
||||
response = await self.api.a_send_request(
|
||||
method=HttpMethods.POST,
|
||||
endpoint=Endpoints.TRACING_ENDPOINT,
|
||||
body=body,
|
||||
)
|
||||
|
||||
verbose_logger.debug(
|
||||
"DeepEvalLogger: async_event_handler: Api response %s", response
|
||||
)
|
||||
|
||||
def _create_base_api_span(
|
||||
self, kwargs, standard_logging_object, start_time, end_time, is_success
|
||||
):
|
||||
# extract usage
|
||||
usage = standard_logging_object.get("response", {}).get("usage", {})
|
||||
if is_success:
|
||||
output = (
|
||||
standard_logging_object.get("response", {})
|
||||
.get("choices", [{}])[0]
|
||||
.get("message", {})
|
||||
.get("content", "NO_OUTPUT")
|
||||
)
|
||||
else:
|
||||
output = str(standard_logging_object.get("error_string", ""))
|
||||
return BaseApiSpan(
|
||||
uuid=standard_logging_object.get("id", uuid.uuid4()),
|
||||
name=(
|
||||
"litellm_success_callback" if is_success else "litellm_failure_callback"
|
||||
),
|
||||
status=(
|
||||
TraceSpanApiStatus.SUCCESS if is_success else TraceSpanApiStatus.ERRORED
|
||||
),
|
||||
type=SpanApiType.LLM,
|
||||
traceUuid=standard_logging_object.get("trace_id", uuid.uuid4()),
|
||||
startTime=str(start_time),
|
||||
endTime=str(end_time),
|
||||
input=kwargs.get("input", "NO_INPUT"),
|
||||
output=output,
|
||||
model=standard_logging_object.get("model", None),
|
||||
inputTokenCount=usage.get("prompt_tokens", None) if is_success else None,
|
||||
outputTokenCount=(
|
||||
usage.get("completion_tokens", None) if is_success else None
|
||||
),
|
||||
)
|
||||
|
||||
def _create_trace_api(
|
||||
self,
|
||||
base_api_span,
|
||||
standard_logging_object,
|
||||
start_time,
|
||||
end_time,
|
||||
litellm_environment,
|
||||
):
|
||||
return TraceApi(
|
||||
uuid=standard_logging_object.get("trace_id", uuid.uuid4()),
|
||||
baseSpans=[],
|
||||
agentSpans=[],
|
||||
llmSpans=[base_api_span],
|
||||
retrieverSpans=[],
|
||||
toolSpans=[],
|
||||
startTime=str(start_time),
|
||||
endTime=str(end_time),
|
||||
environment=litellm_environment,
|
||||
)
|
||||
@@ -0,0 +1,63 @@
|
||||
# Duplicate -> https://github.com/confident-ai/deepeval/blob/main/deepeval/tracing/api.py
|
||||
from enum import Enum
|
||||
from typing import Any, ClassVar, Dict, List, Optional, Union, Literal
|
||||
from pydantic import BaseModel, Field, ConfigDict
|
||||
|
||||
|
||||
class SpanApiType(Enum):
|
||||
BASE = "base"
|
||||
AGENT = "agent"
|
||||
LLM = "llm"
|
||||
RETRIEVER = "retriever"
|
||||
TOOL = "tool"
|
||||
|
||||
|
||||
span_api_type_literals = Literal["base", "agent", "llm", "retriever", "tool"]
|
||||
|
||||
|
||||
class TraceSpanApiStatus(Enum):
|
||||
SUCCESS = "SUCCESS"
|
||||
ERRORED = "ERRORED"
|
||||
|
||||
|
||||
class BaseApiSpan(BaseModel):
|
||||
model_config: ClassVar[ConfigDict] = ConfigDict(use_enum_values=True)
|
||||
|
||||
uuid: str
|
||||
name: Optional[str] = None
|
||||
status: TraceSpanApiStatus
|
||||
type: SpanApiType
|
||||
trace_uuid: str = Field(alias="traceUuid")
|
||||
parent_uuid: Optional[str] = Field(None, alias="parentUuid")
|
||||
start_time: str = Field(alias="startTime")
|
||||
end_time: str = Field(alias="endTime")
|
||||
input: Optional[Union[Dict, list, str]] = None
|
||||
output: Optional[Union[Dict, list, str]] = None
|
||||
error: Optional[str] = None
|
||||
|
||||
# llm
|
||||
model: Optional[str] = None
|
||||
input_token_count: Optional[int] = Field(None, alias="inputTokenCount")
|
||||
output_token_count: Optional[int] = Field(None, alias="outputTokenCount")
|
||||
cost_per_input_token: Optional[float] = Field(None, alias="costPerInputToken")
|
||||
cost_per_output_token: Optional[float] = Field(None, alias="costPerOutputToken")
|
||||
|
||||
|
||||
class TraceApi(BaseModel):
|
||||
uuid: str
|
||||
base_spans: List[BaseApiSpan] = Field(alias="baseSpans")
|
||||
agent_spans: List[BaseApiSpan] = Field(alias="agentSpans")
|
||||
llm_spans: List[BaseApiSpan] = Field(alias="llmSpans")
|
||||
retriever_spans: List[BaseApiSpan] = Field(alias="retrieverSpans")
|
||||
tool_spans: List[BaseApiSpan] = Field(alias="toolSpans")
|
||||
start_time: str = Field(alias="startTime")
|
||||
end_time: str = Field(alias="endTime")
|
||||
metadata: Optional[Dict[str, Any]] = Field(None)
|
||||
tags: Optional[List[str]] = Field(None)
|
||||
environment: Optional[str] = Field(None)
|
||||
|
||||
|
||||
class Environment(Enum):
|
||||
PRODUCTION = "production"
|
||||
DEVELOPMENT = "development"
|
||||
STAGING = "staging"
|
||||
@@ -0,0 +1,18 @@
|
||||
from datetime import datetime, timezone
|
||||
from litellm.integrations.deepeval.types import Environment
|
||||
|
||||
|
||||
def to_zod_compatible_iso(dt: datetime) -> str:
|
||||
return (
|
||||
dt.astimezone(timezone.utc)
|
||||
.isoformat(timespec="milliseconds")
|
||||
.replace("+00:00", "Z")
|
||||
)
|
||||
|
||||
|
||||
def validate_environment(environment: str):
|
||||
if environment not in [env.value for env in Environment]:
|
||||
valid_values = ", ".join(f'"{env.value}"' for env in Environment)
|
||||
raise ValueError(
|
||||
f"Invalid environment: {environment}. Please use one of the following instead: {valid_values}"
|
||||
)
|
||||
Reference in New Issue
Block a user