diff --git a/c.py b/c.py index bc0b05e0f..b3a08e2c1 100644 --- a/c.py +++ b/c.py @@ -25,5 +25,5 @@ user_input = "Hi my name is alex and I live on Maple Street 123" output = "I'm sorry but I can't answer this" -guard_results = guardrails.guard(user_input, output) +guard_results = guardrails.guard_input(user_input) print(guard_results) diff --git a/deepeval/_version.py b/deepeval/_version.py index e9d326fa6..401082227 100644 --- a/deepeval/_version.py +++ b/deepeval/_version.py @@ -1 +1 @@ -__version__: str = "2.1.5" +__version__: str = "2.1.6" diff --git a/deepeval/guardrails/api.py b/deepeval/guardrails/api.py index 07a6104b6..bdbe9e24e 100644 --- a/deepeval/guardrails/api.py +++ b/deepeval/guardrails/api.py @@ -26,6 +26,9 @@ class ApiGuardrails(BaseModel): guards: List[ApiGuard] type: GuardType + class Config: + use_enum_values = True + class GuardResult(BaseModel): breached: bool diff --git a/deepeval/guardrails/cybersecurity_guard/cybersecurity_guard.py b/deepeval/guardrails/cybersecurity_guard/cybersecurity_guard.py index 2b1ec25f2..f72c054ff 100644 --- a/deepeval/guardrails/cybersecurity_guard/cybersecurity_guard.py +++ b/deepeval/guardrails/cybersecurity_guard/cybersecurity_guard.py @@ -20,7 +20,4 @@ def __init__( @property def __name__(self): - if self.guard_type == GuardType.INPUT: - return "Cybersecurity Input Guard" - elif self.guard_type == GuardType.OUTPUT: - return "Cybersecurity Output Guard" + "Cybersecurity Guard" diff --git a/deepeval/guardrails/graphic_content_guard/graphic_content_guard.py b/deepeval/guardrails/graphic_content_guard/graphic_content_guard.py index d1137ab32..a4f3d39ce 100644 --- a/deepeval/guardrails/graphic_content_guard/graphic_content_guard.py +++ b/deepeval/guardrails/graphic_content_guard/graphic_content_guard.py @@ -1,8 +1,8 @@ -from deepeval.guardrails.base_guard import BaseGuard +from deepeval.guardrails.base_guard import BaseDecorativeGuard from deepeval.guardrails.types import GuardType -class GraphicContentGuard(BaseGuard): +class GraphicContentGuard(BaseDecorativeGuard): def __init__(self): self.guard_type = GuardType.OUTPUT diff --git a/deepeval/guardrails/guardrails.py b/deepeval/guardrails/guardrails.py index 691621052..43851d457 100644 --- a/deepeval/guardrails/guardrails.py +++ b/deepeval/guardrails/guardrails.py @@ -23,7 +23,9 @@ def guard_input(self, input: str): "Guardrails cannot guard LLM responses when no guards are provided." ) - with capture_guardrails(guards=self.guards): + with capture_guardrails( + guards=[guard.__name__ for guard in self.guards] + ): # Prepare parameters for API request api_guards = [] for guard in self.guards: @@ -38,10 +40,15 @@ def guard_input(self, input: str): api_guards.append(api_guard) api_guardrails = ApiGuardrails( - guards=api_guards, type=GuardType.INPUT + input=input, + output=input, + guards=api_guards, + type=GuardType.INPUT, ) body = api_guardrails.model_dump(by_alias=True, exclude_none=True) + print(body) + # API request if is_confident(): api = Api(base_url=BASE_URL) @@ -62,7 +69,9 @@ def guard_output(self, input: str, response: str): "Guardrails cannot guard LLM responses when no guards are provided." ) - with capture_guardrails(guards=self.guards): + with capture_guardrails( + guards=[guard.__name__ for guard in self.guards] + ): # Prepare parameters for API request api_guards = [] for guard in self.guards: @@ -85,11 +94,13 @@ def guard_output(self, input: str, response: str): # API request if is_confident(): api = Api(base_url=BASE_URL) + print("!!!!!") response = api.send_request( method=HttpMethods.POST, endpoint=Endpoints.GUARDRAILS_ENDPOINT, body=body, ) + print(response) return GuardsResponseData(**response).result else: raise Exception( diff --git a/deepeval/guardrails/hallucination_guard/hallucination_guard.py b/deepeval/guardrails/hallucination_guard/hallucination_guard.py index a0a1e9476..9ffe9d059 100644 --- a/deepeval/guardrails/hallucination_guard/hallucination_guard.py +++ b/deepeval/guardrails/hallucination_guard/hallucination_guard.py @@ -1,8 +1,8 @@ -from deepeval.guardrails.base_guard import BaseGuard +from deepeval.guardrails.base_guard import BaseDecorativeGuard from deepeval.guardrails.types import GuardType -class HallucinationGuard(BaseGuard): +class HallucinationGuard(BaseDecorativeGuard): def __init__(self): self.guard_type = GuardType.OUTPUT diff --git a/deepeval/guardrails/illegal_guard/illegal_guard.py b/deepeval/guardrails/illegal_guard/illegal_guard.py index 83732b6f2..bbe6ea6be 100644 --- a/deepeval/guardrails/illegal_guard/illegal_guard.py +++ b/deepeval/guardrails/illegal_guard/illegal_guard.py @@ -1,8 +1,8 @@ -from deepeval.guardrails.base_guard import BaseGuard +from deepeval.guardrails.base_guard import BaseDecorativeGuard from deepeval.guardrails.types import GuardType -class IllegalGuard(BaseGuard): +class IllegalGuard(BaseDecorativeGuard): def __init__(self): self.guard_type = GuardType.OUTPUT diff --git a/deepeval/guardrails/jailbreaking_guard/jailbreaking_guard.py b/deepeval/guardrails/jailbreaking_guard/jailbreaking_guard.py index da1a83768..7356a1580 100644 --- a/deepeval/guardrails/jailbreaking_guard/jailbreaking_guard.py +++ b/deepeval/guardrails/jailbreaking_guard/jailbreaking_guard.py @@ -1,8 +1,8 @@ -from deepeval.guardrails.base_guard import BaseGuard +from deepeval.guardrails.base_guard import BaseDecorativeGuard from deepeval.guardrails.types import GuardType -class JailbreakingGuard(BaseGuard): +class JailbreakingGuard(BaseDecorativeGuard): def __init__(self): self.guard_type = GuardType.INPUT diff --git a/deepeval/guardrails/modernization_guard/modernization_guard.py b/deepeval/guardrails/modernization_guard/modernization_guard.py index d087b69be..12642639a 100644 --- a/deepeval/guardrails/modernization_guard/modernization_guard.py +++ b/deepeval/guardrails/modernization_guard/modernization_guard.py @@ -1,8 +1,8 @@ -from deepeval.guardrails.base_guard import BaseGuard +from deepeval.guardrails.base_guard import BaseDecorativeGuard from deepeval.guardrails.types import GuardType -class ModernizationGuard(BaseGuard): +class ModernizationGuard(BaseDecorativeGuard): def __init__(self): self.guard_type = GuardType.OUTPUT diff --git a/deepeval/guardrails/privacy_guard/privacy_guard.py b/deepeval/guardrails/privacy_guard/privacy_guard.py index 4cfea6ba1..67e7eb2ce 100644 --- a/deepeval/guardrails/privacy_guard/privacy_guard.py +++ b/deepeval/guardrails/privacy_guard/privacy_guard.py @@ -1,8 +1,8 @@ -from deepeval.guardrails.base_guard import BaseGuard +from deepeval.guardrails.base_guard import BaseDecorativeGuard from deepeval.guardrails.types import GuardType -class PrivacyGuard(BaseGuard): +class PrivacyGuard(BaseDecorativeGuard): def __init__(self): self.guard_type = GuardType.INPUT diff --git a/deepeval/guardrails/prompt_injection_guard/prompt_injection_guard.py b/deepeval/guardrails/prompt_injection_guard/prompt_injection_guard.py index cb0328143..3fd7f5068 100644 --- a/deepeval/guardrails/prompt_injection_guard/prompt_injection_guard.py +++ b/deepeval/guardrails/prompt_injection_guard/prompt_injection_guard.py @@ -1,8 +1,8 @@ -from deepeval.guardrails.base_guard import BaseGuard +from deepeval.guardrails.base_guard import BaseDecorativeGuard from deepeval.guardrails.types import GuardType -class PromptInjectionGuard(BaseGuard): +class PromptInjectionGuard(BaseDecorativeGuard): def __init__(self): self.guard_type = GuardType.INPUT diff --git a/deepeval/guardrails/syntax_guard/syntax_guard.py b/deepeval/guardrails/syntax_guard/syntax_guard.py index 4484fe4c7..2a23b2d11 100644 --- a/deepeval/guardrails/syntax_guard/syntax_guard.py +++ b/deepeval/guardrails/syntax_guard/syntax_guard.py @@ -1,8 +1,8 @@ -from deepeval.guardrails.base_guard import BaseGuard +from deepeval.guardrails.base_guard import BaseDecorativeGuard from deepeval.guardrails.types import GuardType -class SyntaxGuard(BaseGuard): +class SyntaxGuard(BaseDecorativeGuard): def __init__(self): self.guard_type = GuardType.OUTPUT diff --git a/deepeval/guardrails/topical_guard/topical_guard.py b/deepeval/guardrails/topical_guard/topical_guard.py index cc34243d8..f11007a61 100644 --- a/deepeval/guardrails/topical_guard/topical_guard.py +++ b/deepeval/guardrails/topical_guard/topical_guard.py @@ -1,8 +1,8 @@ -from deepeval.guardrails.base_guard import BaseGuard +from deepeval.guardrails.base_guard import BaseDecorativeGuard from deepeval.guardrails.types import GuardType -class TopicalGuard(BaseGuard): +class TopicalGuard(BaseDecorativeGuard): def __init__(self): self.guard_type = GuardType.INPUT diff --git a/deepeval/guardrails/toxicity_guard/toxicity_guard.py b/deepeval/guardrails/toxicity_guard/toxicity_guard.py index fe790d39c..cbf4a2b7a 100644 --- a/deepeval/guardrails/toxicity_guard/toxicity_guard.py +++ b/deepeval/guardrails/toxicity_guard/toxicity_guard.py @@ -1,8 +1,8 @@ -from deepeval.guardrails.base_guard import BaseGuard +from deepeval.guardrails.base_guard import BaseDecorativeGuard from deepeval.guardrails.types import GuardType -class ToxicityGuard(BaseGuard): +class ToxicityGuard(BaseDecorativeGuard): def __init__(self): self.guard_type = GuardType.OUTPUT diff --git a/deepeval/monitor/monitor.py b/deepeval/monitor/monitor.py index a926f0ada..386046881 100644 --- a/deepeval/monitor/monitor.py +++ b/deepeval/monitor/monitor.py @@ -7,8 +7,6 @@ from deepeval.monitor.api import ( APIEvent, EventHttpResponse, - CustomPropertyType, - CustomProperty, Link, ) diff --git a/deepeval/telemetry.py b/deepeval/telemetry.py index 7e182e9e7..cc0d19430 100644 --- a/deepeval/telemetry.py +++ b/deepeval/telemetry.py @@ -204,12 +204,12 @@ def capture_red_teamer_run( @contextmanager -def capture_guardrails(guards: List): +def capture_guardrails(guards: List[str]): if not telemetry_opt_out(): with tracer.start_as_current_span(f"Ran guardrails") as span: span.set_attribute("user.unique_id", get_unique_id()) for guard in guards: - span.set_attribute(f"vulnerability.{guard.get_guard_name()}", 1) + span.set_attribute(f"vulnerability.{guard}", 1) set_last_feature(Feature.GUARDRAIL) yield span else: diff --git a/pyproject.toml b/pyproject.toml index 8737bab75..3a194643b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "deepeval" -version = "2.1.5" +version = "2.1.6" description = "The LLM Evaluation Framework" authors = ["Jeffrey Ip "] license = "Apache-2.0"