diff --git a/.gitignore b/.gitignore index 00d7b08..ccf5e54 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,5 @@ pip-log.txt .metaflow .metaflow/ .ipynb_checkpoints + +venv \ No newline at end of file diff --git a/aimon/extensions/__init__.py b/aimon/extensions/__init__.py new file mode 100644 index 0000000..fb3a37e --- /dev/null +++ b/aimon/extensions/__init__.py @@ -0,0 +1 @@ +from .react import ReactConfig, React, Framework diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py new file mode 100644 index 0000000..00b66b1 --- /dev/null +++ b/aimon/extensions/react.py @@ -0,0 +1,210 @@ +import os +from enum import Enum +from aimon import Client +from typing import Optional +from dataclasses import dataclass + +import random +import string + +def generate_random_string(length): + """Generates a random string of letters and digits.""" + characters = string.ascii_letters + string.digits + return ''.join(random.choice(characters) for i in range(length)) + + +class Framework(Enum): + LLAMAINDEX = "LlamaIndex" + LANGCHAIN = "LangChain" + HAYSTACK = "Haystack" + NONE = None + +@dataclass +class ReactConfig: + """ + Configuration class for the React configuration settings. + + Attributes: + publish (bool): Flag indicating whether to publish the results to app.aimon.ai + max_attempts (int): Maximum number of ReAct attempts + hallucination_threshold (float): Threshold value to determine hallucination behavior. Defaults to 0.5. + framework (Optional[Framework]): Optional framework configuration. Defaults to None. + aimon_api_key (Optional[str]): API key for AIMon integration. If not provided, it attempts to retrieve from environment variable "AIMON_API_KEY". + model_name (Optional[str]): Name of the model to be used. Defaults to a string based on "aimon-react-model" concatenated with a random string. + application_name (Optional[str]): Name of the application. Defaults to a string based on "aimon-react-application" concatenated with a random string. + + Methods: + None + """ + publish: bool + max_attempts: int + hallucination_threshold: float = 0.5 + framework: Optional[Framework] = None + aimon_api_key: Optional[str] = os.getenv("AIMON_API_KEY") + model_name: Optional[str] = "aimon-react-model" + generate_random_string(5) + application_name: Optional[str] = "aimon-react-application" + generate_random_string(5) + +class React: + + ## Initailze the AIMon Client here + def __init__(self, llm_app, react_configuration, context_extractor): + + self.llm_app = llm_app + self.context_extractor = context_extractor + self.react_configuration = react_configuration + self.client = Client(auth_header="Bearer {}".format(self.react_configuration.aimon_api_key)) + + def create_payload(self, context, user_query, user_instructions, generated_text): + + aimon_payload = { + 'context':context, + 'user_query':user_query, + 'generated_text':generated_text, + 'instructions':user_instructions, + } + + aimon_payload['publish'] = self.react_configuration.publish + aimon_payload['config'] = { 'hallucination': {'detector_name': 'default'}, + 'instruction_adherence': {'detector_name': 'default'},} + + if self.react_configuration.publish: + aimon_payload['application_name'] = self.react_configuration.application_name + aimon_payload['model_name'] = self.react_configuration.model_name + + return aimon_payload + + def detect_aimon_response(self,aimon_payload): + + try: + detect_response = self.client.inference.detect(body=[aimon_payload]) + # Check if the response is a list + if isinstance(detect_response, list) and len(detect_response) > 0: + detect_result = detect_response[0] + elif isinstance(detect_response, dict): + detect_result = detect_response # Single dict response + else: + raise ValueError("Unexpected response format from detect API: {}".format(detect_response)) + except Exception as e: + # Log the error and raise it + print(f"Error during detection: {e}") + raise + + return detect_result + + ## ReAct -> Reason and Act + def react(self, user_query, user_instructions): + """ + AIMon-ReAct -> Reason and Act with AIMon + + where ReAct score: + 1.0 if after n attempts of ReAct, the LLM follows instructions and generates a non hallucinated response. + 0.5 if after n attempts of ReAct, either the LLM response follows user_instructions or is under hallucination threshold [BUT NOT BOTH]. + 0.0 otherwise. + """ + result = { + 'responses': [], + 'hallucination_scores': [], + 'adherence': [], + 'react_score': 0.0 ## 0.0 by assumption + } + + llm_response = self.llm_app(user_query, user_instructions, reprompted_flag=False) + + context = self.context_extractor(user_query, user_instructions, llm_response) + + if isinstance(llm_response, str): + generated_text = llm_response + elif self.react_configuration.framework.value=="LlamaIndex" or hasattr(llm_response, 'response'): + generated_text = llm_response.response + else: + generated_text = llm_response + + aimon_payload = self.create_payload(context, user_query, user_instructions, generated_text) + + detect_response = self.detect_aimon_response(aimon_payload) + + hallucination_score = detect_response.hallucination['score'] + + failed_instruction_counter = 0 + ia_feedback = "" + ## Loop to check for failed instructions + for x in detect_response.instruction_adherence['results']: + if x['adherence'] == False: + failed_instruction_counter+=1 + adherence = False + ia_feedback += f"For the instruction {x['instruction']}, the adherence detector evaluated that {x['detailed_explanation']}." + if failed_instruction_counter == 0: + adherence = True + if ia_feedback=="": + ia_feedback="All the instructions were complied with." + ## Baseline + result['adherence'].append(adherence) + result['responses'].append(generated_text) + result['hallucination_scores'].append(hallucination_score) + + for _ in range(self.react_configuration.max_attempts): + + ## Check whether the hallucination score is greater than the required threshold OR if any of the supplied instructions are not complied with + if self.react_configuration.hallucination_threshold > 0 and \ + (hallucination_score > self.react_configuration.hallucination_threshold or failed_instruction_counter>0): + + llm_response = self.llm_app(user_query, + user_instructions, + reprompted_flag=True, + last_llm_response = generated_text, + hallucination_score=hallucination_score, + ia_feedback = ia_feedback) + + context = self.context_extractor(user_query, user_instructions, llm_response) + + ## Generated text for LLM Response, if the user employs the LlamaIndex framework + if self.react_configuration.framework.value=="LlamaIndex" or llm_response.response: + generated_text = llm_response.response + else: + generated_text = llm_response + + new_aimon_payload = self.create_payload(context, user_query, user_instructions, generated_text) + + detect_response = self.detect_aimon_response(new_aimon_payload) + + hallucination_score = detect_response.hallucination['score'] + + ia_feedback = "" + failed_instruction_counter = 0 + + for x in detect_response.instruction_adherence['results']: + if x['adherence'] == False: + failed_instruction_counter+=1 + adherence = False + ia_feedback += f"For the instruction {x['instruction']}, the adherence detector evaluated that {x['detailed_explanation']}." + if failed_instruction_counter == 0: + adherence = True + if ia_feedback=="": + ia_feedback="All the instructions were complied with." + + result['adherence'].append(adherence) + result['responses'].append(generated_text) + result['hallucination_scores'].append(hallucination_score) + + else: + break + + ## The hallucination score and adherence here are from the last attempt of the max attempts + if hallucination_score > self.react_configuration.hallucination_threshold and adherence==False: + # result['responses'].append(f"Even after {self.react_configuration.max_attempts} attempts of AIMon react, the LLM neither adheres to the user instructions, nor generates a response that is not hallucinated. Final LLM response: {generated_text}") + # result['hallucination_scores'].append(hallucination_score) + result['react_score']=0.0 + elif hallucination_score > self.react_configuration.hallucination_threshold and adherence==True: + # result['responses'].append(f"Although the LLM adheres to the user instructions, the generated response, even after {self.react_configuration.max_attempts} attempts of AIMon ReAct is still hallucinated. Final LLM response: {generated_text}") + # result['hallucination_scores'].append(hallucination_score) + result['react_score']=0.5 + elif hallucination_score <= self.react_configuration.hallucination_threshold and adherence==False: + # result['responses'].append(f"Although the LLM generates a non-hallucinated response, it fails to adhere to the user instructions, even after {self.react_configuration.max_attempts} attempts of AIMon ReAct. Final LLM response: {generated_text}") + # result['hallucination_scores'].append(hallucination_score) + result['react_score']=0.5 + else: + # result['responses'].append(f"This response is below the hallucination threshold and adheres to the user instructions. Response {generated_text}") + # result['hallucination_scores'].append(hallucination_score) + result['react_score']=1.0 + + return result \ No newline at end of file