From 99f09e3372fc5c03ba03d4b5a5bb9c20d45f1e2b Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Sat, 21 Dec 2024 00:52:28 +0530 Subject: [PATCH 01/24] A new directory created for reprompting extension --- aimon/extensions/__init__.py | 0 aimon/extensions/reprompt.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 aimon/extensions/__init__.py create mode 100644 aimon/extensions/reprompt.py diff --git a/aimon/extensions/__init__.py b/aimon/extensions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimon/extensions/reprompt.py b/aimon/extensions/reprompt.py new file mode 100644 index 0000000..e69de29 From 1bd78d6c652f5e96d0280e1951d9152f42f92e11 Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Tue, 24 Dec 2024 01:56:16 +0530 Subject: [PATCH 02/24] First iteration of AIMon ReAct (Reason and Act) --- .gitignore | 2 ++ aimon/extensions/__init__.py | 1 + aimon/extensions/react.py | 62 ++++++++++++++++++++++++++++++++++++ aimon/extensions/reprompt.py | 0 4 files changed, 65 insertions(+) create mode 100644 aimon/extensions/react.py delete mode 100644 aimon/extensions/reprompt.py diff --git a/.gitignore b/.gitignore index 00d7b08..ccf5e54 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,5 @@ pip-log.txt .metaflow .metaflow/ .ipynb_checkpoints + +venv \ No newline at end of file diff --git a/aimon/extensions/__init__.py b/aimon/extensions/__init__.py index e69de29..9924cc9 100644 --- a/aimon/extensions/__init__.py +++ b/aimon/extensions/__init__.py @@ -0,0 +1 @@ +from .react import ReactConfig, react diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py new file mode 100644 index 0000000..f3a5c54 --- /dev/null +++ b/aimon/extensions/react.py @@ -0,0 +1,62 @@ +from aimon import Detect +from dataclasses import dataclass + +@dataclass +class ReactConfig: + publish: bool + model_name: str + max_attempts: int + aimon_api_key: str + application_name: str + values_returned: list[str] + hallucination_threshold: float + aimon_config: dict[str, dict[str, str]] + +## ReAct -> Reason and Act + +def react( llm_app, + user_query, + user_instructions, + context_extractor, + react_configuration, + ): + + detect = Detect(values_returned = react_configuration.values_returned, + api_key = react_configuration.aimon_api_key, + config = react_configuration.aimon_config, + publish = react_configuration.publish, + application_name = react_configuration.application_name, + model_name = react_configuration.model_name, + ) + + llm_response = llm_app(user_query, reprompted_flag=False) + + ## Decorating the context_extractor function with AIMon's "detect" + context_extractor = detect(context_extractor) + + _, _, _, query_result, aimon_response = context_extractor(llm_response, user_query, user_instructions) + + for _ in range(react_configuration.max_attempts): + + if aimon_response.detect_response.hallucination['score'] > react_configuration.hallucination_threshold: + llm_response = llm_app(user_query, reprompted_flag=True) + _, _, _, query_result, aimon_response = context_extractor(llm_response, user_query, user_instructions) + + return query_result + + +## To do: +## Add instruction adherence logic in the next iteration + + +## llm_app is a function that has both conservative and creative LLMs to its access +## returns the LLM's response to the user's query + +## Template for llm_app function +# def llm_app(user_query, reprompted_flag=False): +# creative_llm: function +# conservative_llm: function +# if reprompted_flag==False: +# return creative_llm.query(user_query) +# else: +# return conservative_llm.query(user_query) \ No newline at end of file diff --git a/aimon/extensions/reprompt.py b/aimon/extensions/reprompt.py deleted file mode 100644 index e69de29..0000000 From 560324f10109509b425e3a9032231a42813250d7 Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Tue, 24 Dec 2024 02:08:24 +0530 Subject: [PATCH 03/24] updated parameters for context_extractor --- aimon/extensions/react.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py index f3a5c54..2ae753c 100644 --- a/aimon/extensions/react.py +++ b/aimon/extensions/react.py @@ -34,13 +34,13 @@ def react( llm_app, ## Decorating the context_extractor function with AIMon's "detect" context_extractor = detect(context_extractor) - _, _, _, query_result, aimon_response = context_extractor(llm_response, user_query, user_instructions) + _, _, _, query_result, aimon_response = context_extractor(user_query, user_instructions, llm_response) for _ in range(react_configuration.max_attempts): if aimon_response.detect_response.hallucination['score'] > react_configuration.hallucination_threshold: llm_response = llm_app(user_query, reprompted_flag=True) - _, _, _, query_result, aimon_response = context_extractor(llm_response, user_query, user_instructions) + _, _, _, query_result, aimon_response = context_extractor(user_query, user_instructions, llm_response) return query_result From 12cc16132c9288c206bd2fe08632b2226e039531 Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Tue, 24 Dec 2024 22:25:03 +0530 Subject: [PATCH 04/24] added the instruction adherence logic to AIMon ReAct --- aimon/extensions/react.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py index 2ae753c..d48f5f2 100644 --- a/aimon/extensions/react.py +++ b/aimon/extensions/react.py @@ -29,7 +29,7 @@ def react( llm_app, model_name = react_configuration.model_name, ) - llm_response = llm_app(user_query, reprompted_flag=False) + llm_response = llm_app(user_query, user_instructions, reprompted_flag=False) ## Decorating the context_extractor function with AIMon's "detect" context_extractor = detect(context_extractor) @@ -38,25 +38,33 @@ def react( llm_app, for _ in range(react_configuration.max_attempts): - if aimon_response.detect_response.hallucination['score'] > react_configuration.hallucination_threshold: - llm_response = llm_app(user_query, reprompted_flag=True) + failed_instructions = [] + ## Loop to check for failed instructions + for x in aimon_response.detect_response.instruction_adherence['results']: + if x['adherence'] == False: + failed_instructions.append(x['instruction']) + + ## Check whether the hallucination score is greater than the required threshold OR if any of the supplied instructions are not complied with + if react_configuration.hallucination_threshold > 0 and \ + (aimon_response.detect_response.hallucination['score'] > react_configuration.hallucination_threshold or len(failed_instructions)>0): + + llm_response = llm_app(user_query, user_instructions, reprompted_flag=True) + _, _, _, query_result, aimon_response = context_extractor(user_query, user_instructions, llm_response) return query_result -## To do: -## Add instruction adherence logic in the next iteration ## llm_app is a function that has both conservative and creative LLMs to its access ## returns the LLM's response to the user's query ## Template for llm_app function -# def llm_app(user_query, reprompted_flag=False): -# creative_llm: function -# conservative_llm: function +# def llm_app(user_query, user_instructions, reprompted_flag=False): +# from aimon_llamaindex import get_response # if reprompted_flag==False: -# return creative_llm.query(user_query) +# return get_response(user_query, retriever, llm_creative) # else: -# return conservative_llm.query(user_query) \ No newline at end of file +# llm_conservative.system_prompt += user_instructions +# return get_response(user_query, retriever, llm_conservative) \ No newline at end of file From e3beac6048b6536b9ef26d3edc4b9d4eff6ec5c0 Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Tue, 24 Dec 2024 22:45:23 +0530 Subject: [PATCH 05/24] passing hallucination score to modify the new prompt dynamically --- aimon/extensions/react.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py index d48f5f2..2d40c9f 100644 --- a/aimon/extensions/react.py +++ b/aimon/extensions/react.py @@ -44,11 +44,13 @@ def react( llm_app, if x['adherence'] == False: failed_instructions.append(x['instruction']) + hallucination_score = aimon_response.detect_response.hallucination['score'] + ## Check whether the hallucination score is greater than the required threshold OR if any of the supplied instructions are not complied with if react_configuration.hallucination_threshold > 0 and \ - (aimon_response.detect_response.hallucination['score'] > react_configuration.hallucination_threshold or len(failed_instructions)>0): + (hallucination_score > react_configuration.hallucination_threshold or len(failed_instructions)>0): - llm_response = llm_app(user_query, user_instructions, reprompted_flag=True) + llm_response = llm_app(user_query, user_instructions, reprompted_flag=True, hallucination_score=hallucination_score) _, _, _, query_result, aimon_response = context_extractor(user_query, user_instructions, llm_response) From e354ff54e8a76d61c6e3b91e08e2448b67962fbb Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Tue, 24 Dec 2024 22:52:47 +0530 Subject: [PATCH 06/24] added condition and logic to handle the case when the response is still hallucinated after n number of max attemps --- aimon/extensions/react.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py index 2d40c9f..3fbc5a1 100644 --- a/aimon/extensions/react.py +++ b/aimon/extensions/react.py @@ -54,19 +54,7 @@ def react( llm_app, _, _, _, query_result, aimon_response = context_extractor(user_query, user_instructions, llm_response) - return query_result + if hallucination_score > react_configuration.hallucination_threshold: + return f"The generated LLM response, even after {react_configuration.max_attempts} attempts of ReAct is still hallucinated. The response: {query_result}" - - - -## llm_app is a function that has both conservative and creative LLMs to its access -## returns the LLM's response to the user's query - -## Template for llm_app function -# def llm_app(user_query, user_instructions, reprompted_flag=False): -# from aimon_llamaindex import get_response -# if reprompted_flag==False: -# return get_response(user_query, retriever, llm_creative) -# else: -# llm_conservative.system_prompt += user_instructions -# return get_response(user_query, retriever, llm_conservative) \ No newline at end of file + return query_result \ No newline at end of file From f2049d06a66e25bd8b4314c4f346a8bbc823021f Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Fri, 27 Dec 2024 03:01:39 +0530 Subject: [PATCH 07/24] class React --- aimon/extensions/__init__.py | 2 +- aimon/extensions/react.py | 112 +++++++++++++++++++++++------------ 2 files changed, 74 insertions(+), 40 deletions(-) diff --git a/aimon/extensions/__init__.py b/aimon/extensions/__init__.py index 9924cc9..99bcce2 100644 --- a/aimon/extensions/__init__.py +++ b/aimon/extensions/__init__.py @@ -1 +1 @@ -from .react import ReactConfig, react +from .react import ReactConfig, React diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py index 3fbc5a1..b4b113c 100644 --- a/aimon/extensions/react.py +++ b/aimon/extensions/react.py @@ -1,60 +1,94 @@ -from aimon import Detect +from aimon import Client, Detect from dataclasses import dataclass @dataclass class ReactConfig: publish: bool + async_mode:bool model_name: str max_attempts: int aimon_api_key: str application_name: str - values_returned: list[str] hallucination_threshold: float - aimon_config: dict[str, dict[str, str]] + framework:str = None -## ReAct -> Reason and Act +class React: + + ## Initailze the AIMon Client here + def __init__(self, llm_app, react_configuration, context_extractor): + + self.llm_app = llm_app + self.context_extractor = context_extractor + self.react_configuration = react_configuration + self.client = Client(auth_header="Bearer {}".format(self.react_configuration.aimon_api_key)) + + def create_payload(self, context, user_query, user_instructions, generated_text): + + aimon_payload = { + 'context':context, + 'user_query':user_query, + 'generated_text':generated_text, + 'instructions':user_instructions, + } + + aimon_payload['publish'] = self.react_configuration.publish + aimon_payload['async_mode'] = self.react_configuration.async_mode + aimon_payload['config'] = { 'hallucination': {'detector_name': 'default'}, + 'instruction_adherence': {'detector_name': 'default'},} + + if self.react_configuration.publish: + aimon_payload['application_name'] = self.react_configuration.application_name + aimon_payload['model_name'] = self.react_configuration.model_name + + return aimon_payload -def react( llm_app, - user_query, - user_instructions, - context_extractor, - react_configuration, - ): + + ## ReAct -> Reason and Act + def react(self, user_query, user_instructions,): + + llm_response = self.llm_app(user_query, user_instructions, reprompted_flag=False) + + context = self.context_extractor(user_query, user_instructions, llm_response) + + ## Generated text for LLM Response, if the user employs the LlamaIndex framework + if llm_response.response or self.react_configuration.framework=="llamaindex": + generated_text = llm_response.response + else: + generated_text = llm_response - detect = Detect(values_returned = react_configuration.values_returned, - api_key = react_configuration.aimon_api_key, - config = react_configuration.aimon_config, - publish = react_configuration.publish, - application_name = react_configuration.application_name, - model_name = react_configuration.model_name, - ) - - llm_response = llm_app(user_query, user_instructions, reprompted_flag=False) - - ## Decorating the context_extractor function with AIMon's "detect" - context_extractor = detect(context_extractor) + aimon_payload = self.create_payload(context, user_query, user_instructions, generated_text) - _, _, _, query_result, aimon_response = context_extractor(user_query, user_instructions, llm_response) + detect_response = self.client.inference.detect(body=[aimon_payload]) + + for _ in range(self.react_configuration.max_attempts): - for _ in range(react_configuration.max_attempts): + failed_instructions = [] + ## Loop to check for failed instructions + for x in detect_response.instruction_adherence['results']: + if x['adherence'] == False: + failed_instructions.append(x['instruction']) - failed_instructions = [] - ## Loop to check for failed instructions - for x in aimon_response.detect_response.instruction_adherence['results']: - if x['adherence'] == False: - failed_instructions.append(x['instruction']) + hallucination_score = detect_response.hallucination['score'] - hallucination_score = aimon_response.detect_response.hallucination['score'] + ## Check whether the hallucination score is greater than the required threshold OR if any of the supplied instructions are not complied with + if self.react_configuration.hallucination_threshold > 0 and \ + (hallucination_score > self.react_configuration.hallucination_threshold or len(failed_instructions)>0): + + llm_response = self.llm_app(user_query, user_instructions, reprompted_flag=True, hallucination_score=hallucination_score) + + context = self.context_extractor(user_query, user_instructions, llm_response) + + ## Generated text for LLM Response, if the user employs the LlamaIndex framework + if llm_response.response or self.react_configuration.framework=="llamaindex": + generated_text = llm_response.response + else: + generated_text = llm_response - ## Check whether the hallucination score is greater than the required threshold OR if any of the supplied instructions are not complied with - if react_configuration.hallucination_threshold > 0 and \ - (hallucination_score > react_configuration.hallucination_threshold or len(failed_instructions)>0): - - llm_response = llm_app(user_query, user_instructions, reprompted_flag=True, hallucination_score=hallucination_score) + new_aimon_payload = self.create_payload(context, user_query, user_instructions, generated_text) - _, _, _, query_result, aimon_response = context_extractor(user_query, user_instructions, llm_response) + detect_response = self.client.inference.detect(body=[new_aimon_payload]) - if hallucination_score > react_configuration.hallucination_threshold: - return f"The generated LLM response, even after {react_configuration.max_attempts} attempts of ReAct is still hallucinated. The response: {query_result}" + if hallucination_score > self.react_configuration.hallucination_threshold: + return f"The generated LLM response, even after {self.react_configuration.max_attempts} attempts of ReAct is still hallucinated. The response: {generated_text}" - return query_result \ No newline at end of file + return generated_text \ No newline at end of file From 0eb154d5937a3393ddc0ada7f82e2799e1ae268a Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Fri, 27 Dec 2024 04:10:05 +0530 Subject: [PATCH 08/24] added more methods to the class React --- aimon/extensions/react.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py index b4b113c..6f5c46f 100644 --- a/aimon/extensions/react.py +++ b/aimon/extensions/react.py @@ -42,6 +42,23 @@ def create_payload(self, context, user_query, user_instructions, generated_text) return aimon_payload + def detect_aimon_response(self,aimon_payload): + + try: + detect_response = self.client.inference.detect(body=[aimon_payload]) + # Check if the response is a list + if isinstance(detect_response, list) and len(detect_response) > 0: + detect_result = detect_response[0] + elif isinstance(detect_response, dict): + detect_result = detect_response # Single dict response + else: + raise ValueError("Unexpected response format from detect API: {}".format(detect_response)) + except Exception as e: + # Log the error and raise it + print(f"Error during detection: {e}") + raise + + return detect_result ## ReAct -> Reason and Act def react(self, user_query, user_instructions,): @@ -58,7 +75,7 @@ def react(self, user_query, user_instructions,): aimon_payload = self.create_payload(context, user_query, user_instructions, generated_text) - detect_response = self.client.inference.detect(body=[aimon_payload]) + detect_response = self.detect_aimon_response(aimon_payload) for _ in range(self.react_configuration.max_attempts): @@ -86,7 +103,8 @@ def react(self, user_query, user_instructions,): new_aimon_payload = self.create_payload(context, user_query, user_instructions, generated_text) - detect_response = self.client.inference.detect(body=[new_aimon_payload]) + detect_response = self.detect_aimon_response(new_aimon_payload) + if hallucination_score > self.react_configuration.hallucination_threshold: return f"The generated LLM response, even after {self.react_configuration.max_attempts} attempts of ReAct is still hallucinated. The response: {generated_text}" From b6cce903c0a3c57bc25fbd3e066e88ec32bc6b2f Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Sat, 28 Dec 2024 00:09:23 +0530 Subject: [PATCH 09/24] modified some parameters in the ReactConfig dataclass to be optional --- aimon/extensions/react.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py index 6f5c46f..64d6071 100644 --- a/aimon/extensions/react.py +++ b/aimon/extensions/react.py @@ -1,16 +1,16 @@ +from typing import Optional from aimon import Client, Detect from dataclasses import dataclass @dataclass class ReactConfig: publish: bool - async_mode:bool - model_name: str max_attempts: int aimon_api_key: str - application_name: str hallucination_threshold: float - framework:str = None + framework: Optional[str] = None + model_name: Optional[str] = "aimon-react-model" + application_name: Optional[str] = "aimon-react-application" class React: @@ -32,7 +32,6 @@ def create_payload(self, context, user_query, user_instructions, generated_text) } aimon_payload['publish'] = self.react_configuration.publish - aimon_payload['async_mode'] = self.react_configuration.async_mode aimon_payload['config'] = { 'hallucination': {'detector_name': 'default'}, 'instruction_adherence': {'detector_name': 'default'},} From aa4e6f4388ef5eed5811e49c4926376827801cd7 Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Sat, 28 Dec 2024 00:14:48 +0530 Subject: [PATCH 10/24] removed Detect from aimon import --- aimon/extensions/react.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py index 64d6071..789d5e9 100644 --- a/aimon/extensions/react.py +++ b/aimon/extensions/react.py @@ -1,5 +1,5 @@ +from aimon import Client from typing import Optional -from aimon import Client, Detect from dataclasses import dataclass @dataclass From 4cd39f22354947abab12293b2fb5b68914ef8a90 Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Sat, 11 Jan 2025 20:53:14 +0530 Subject: [PATCH 11/24] updated the react method to returns a list of lists where each list comprises of llm_responses and respecitve hallucination score. --- aimon/extensions/react.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py index 789d5e9..a2227f1 100644 --- a/aimon/extensions/react.py +++ b/aimon/extensions/react.py @@ -62,6 +62,8 @@ def detect_aimon_response(self,aimon_payload): ## ReAct -> Reason and Act def react(self, user_query, user_instructions,): + result = [] + llm_response = self.llm_app(user_query, user_instructions, reprompted_flag=False) context = self.context_extractor(user_query, user_instructions, llm_response) @@ -75,6 +77,10 @@ def react(self, user_query, user_instructions,): aimon_payload = self.create_payload(context, user_query, user_instructions, generated_text) detect_response = self.detect_aimon_response(aimon_payload) + + hallucination_score = detect_response.hallucination['score'] + + result.append([generated_text, hallucination_score]) for _ in range(self.react_configuration.max_attempts): @@ -84,8 +90,6 @@ def react(self, user_query, user_instructions,): if x['adherence'] == False: failed_instructions.append(x['instruction']) - hallucination_score = detect_response.hallucination['score'] - ## Check whether the hallucination score is greater than the required threshold OR if any of the supplied instructions are not complied with if self.react_configuration.hallucination_threshold > 0 and \ (hallucination_score > self.react_configuration.hallucination_threshold or len(failed_instructions)>0): @@ -104,8 +108,14 @@ def react(self, user_query, user_instructions,): detect_response = self.detect_aimon_response(new_aimon_payload) + hallucination_score = detect_response.hallucination['score'] + + result.append([generated_text, hallucination_score]) + + else: + break if hallucination_score > self.react_configuration.hallucination_threshold: - return f"The generated LLM response, even after {self.react_configuration.max_attempts} attempts of ReAct is still hallucinated. The response: {generated_text}" + result.append([f"The generated LLM response, even after {self.react_configuration.max_attempts} attempts of AIMon ReAct is still hallucinated. Final LLM response: {generated_text}", hallucination_score]) - return generated_text \ No newline at end of file + return result \ No newline at end of file From 4e10bf9094306599f099e7ab90d1a419923417ca Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Tue, 14 Jan 2025 00:47:32 +0530 Subject: [PATCH 12/24] returning a result dictionary consiting of a list of response strings, a list of hallucination scores and a boolean value for adhered instuctions --- aimon/extensions/react.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py index a2227f1..200801d 100644 --- a/aimon/extensions/react.py +++ b/aimon/extensions/react.py @@ -62,7 +62,11 @@ def detect_aimon_response(self,aimon_payload): ## ReAct -> Reason and Act def react(self, user_query, user_instructions,): - result = [] + result = { + 'response': [], + 'hscore':[], + 'adherence':True ## True by default + } llm_response = self.llm_app(user_query, user_instructions, reprompted_flag=False) @@ -80,19 +84,27 @@ def react(self, user_query, user_instructions,): hallucination_score = detect_response.hallucination['score'] - result.append([generated_text, hallucination_score]) - + # result.append([generated_text, hallucination_score]) + result['response'].append(generated_text) + result['hscore'].append(hallucination_score) + for _ in range(self.react_configuration.max_attempts): failed_instructions = [] + failed_instruction_counter = 0 ## Loop to check for failed instructions for x in detect_response.instruction_adherence['results']: if x['adherence'] == False: + failed_instruction_counter+=1 failed_instructions.append(x['instruction']) + result['adherence']=False + + if failed_instruction_counter == 0: + result['adherence']=True ## Check whether the hallucination score is greater than the required threshold OR if any of the supplied instructions are not complied with if self.react_configuration.hallucination_threshold > 0 and \ - (hallucination_score > self.react_configuration.hallucination_threshold or len(failed_instructions)>0): + (hallucination_score > self.react_configuration.hallucination_threshold or failed_instruction_counter>0): llm_response = self.llm_app(user_query, user_instructions, reprompted_flag=True, hallucination_score=hallucination_score) @@ -110,12 +122,14 @@ def react(self, user_query, user_instructions,): hallucination_score = detect_response.hallucination['score'] - result.append([generated_text, hallucination_score]) + result['response'].append(generated_text) + result['hscore'].append(hallucination_score) else: break if hallucination_score > self.react_configuration.hallucination_threshold: - result.append([f"The generated LLM response, even after {self.react_configuration.max_attempts} attempts of AIMon ReAct is still hallucinated. Final LLM response: {generated_text}", hallucination_score]) - + result['response'].append(f"The generated LLM response, even after {self.react_configuration.max_attempts} attempts of AIMon ReAct is still hallucinated. Final LLM response: {generated_text}") + result['hscore'].append(hallucination_score) + return result \ No newline at end of file From 92e91f9781c4a5c86fc13056a02cc66437648972 Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Tue, 14 Jan 2025 01:24:30 +0530 Subject: [PATCH 13/24] updated result logic with 4 edge cases --- aimon/extensions/react.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py index 200801d..81e4245 100644 --- a/aimon/extensions/react.py +++ b/aimon/extensions/react.py @@ -128,8 +128,18 @@ def react(self, user_query, user_instructions,): else: break - if hallucination_score > self.react_configuration.hallucination_threshold: - result['response'].append(f"The generated LLM response, even after {self.react_configuration.max_attempts} attempts of AIMon ReAct is still hallucinated. Final LLM response: {generated_text}") + + if hallucination_score > self.react_configuration.hallucination_threshold and result['adherence']==False: + result['response'].append(f"Even after {self.react_configuration.max_attempts} attempts of AIMon react, the LLM neither adheres to the user instructions, nor generates a response that is not hallucinated. Final LLM response: {generated_text}") result['hscore'].append(hallucination_score) - + elif hallucination_score > self.react_configuration.hallucination_threshold and result['adherence']==True: + result['response'].append(f"Although the LLM adheres to the user instructions, the generated response, even after {self.react_configuration.max_attempts} attempts of AIMon ReAct is still hallucinated. Final LLM response: {generated_text}") + result['hscore'].append(hallucination_score) + elif hallucination_score <= self.react_configuration.hallucination_threshold and result['adherence']==False: + result['response'].append(f"Although the LLM generates a non-hallucinated response, it fails to adhere to the user instructions, even after {self.react_configuration.max_attempts} attempts of AIMon ReAct. Final LLM response: {generated_text}") + result['hscore'].append(hallucination_score) + else: + result['response'].append(generated_text) + result['hscore'].append(hallucination_score) + return result \ No newline at end of file From 94e6af4a85cca1e2817c3b89961eb40b414ae9ac Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Tue, 14 Jan 2025 01:44:56 +0530 Subject: [PATCH 14/24] introduced react score and some assumptions. --- aimon/extensions/react.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py index 81e4245..2ff7109 100644 --- a/aimon/extensions/react.py +++ b/aimon/extensions/react.py @@ -61,13 +61,21 @@ def detect_aimon_response(self,aimon_payload): ## ReAct -> Reason and Act def react(self, user_query, user_instructions,): - + """ + AIMon-ReAct -> Reason and Act with AIMon + + where ReAct score: + 1 if after n attempts of ReAct, the LLM follows instructions and generates a non hallucinated response. + 0.5 if after n attempts of ReAct, either the LLM response follows user_instructions or is under hallucination threshold [BUT NOT BOTH]. + 0 otherwise. + """ result = { - 'response': [], - 'hscore':[], - 'adherence':True ## True by default + 'responses': [], + 'hallucination_scores': [], + 'adherence': False, ## False by assumption + 'react_score': 0 ## 0 by assumption } - + llm_response = self.llm_app(user_query, user_instructions, reprompted_flag=False) context = self.context_extractor(user_query, user_instructions, llm_response) @@ -127,19 +135,23 @@ def react(self, user_query, user_instructions,): else: break - - + + if hallucination_score > self.react_configuration.hallucination_threshold and result['adherence']==False: result['response'].append(f"Even after {self.react_configuration.max_attempts} attempts of AIMon react, the LLM neither adheres to the user instructions, nor generates a response that is not hallucinated. Final LLM response: {generated_text}") result['hscore'].append(hallucination_score) + result['react_score']=0 elif hallucination_score > self.react_configuration.hallucination_threshold and result['adherence']==True: result['response'].append(f"Although the LLM adheres to the user instructions, the generated response, even after {self.react_configuration.max_attempts} attempts of AIMon ReAct is still hallucinated. Final LLM response: {generated_text}") result['hscore'].append(hallucination_score) + result['react_score']=0.5 elif hallucination_score <= self.react_configuration.hallucination_threshold and result['adherence']==False: result['response'].append(f"Although the LLM generates a non-hallucinated response, it fails to adhere to the user instructions, even after {self.react_configuration.max_attempts} attempts of AIMon ReAct. Final LLM response: {generated_text}") result['hscore'].append(hallucination_score) + result['react_score']=0.5 else: - result['response'].append(generated_text) + result['response'].append(f"This response is below the hallucination threshold and adheres to the user instructions. Response {generated_text}") result['hscore'].append(hallucination_score) + result['react_score']=1 return result \ No newline at end of file From 6082066302e93c0ce4780978e9ed15ba84cd6c04 Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Tue, 14 Jan 2025 01:54:28 +0530 Subject: [PATCH 15/24] updated keys in the result dictionary --- aimon/extensions/react.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py index 2ff7109..0b66d79 100644 --- a/aimon/extensions/react.py +++ b/aimon/extensions/react.py @@ -93,8 +93,8 @@ def react(self, user_query, user_instructions,): hallucination_score = detect_response.hallucination['score'] # result.append([generated_text, hallucination_score]) - result['response'].append(generated_text) - result['hscore'].append(hallucination_score) + result['responses'].append(generated_text) + result['hallucination_scores'].append(hallucination_score) for _ in range(self.react_configuration.max_attempts): @@ -130,28 +130,28 @@ def react(self, user_query, user_instructions,): hallucination_score = detect_response.hallucination['score'] - result['response'].append(generated_text) - result['hscore'].append(hallucination_score) + result['responses'].append(generated_text) + result['hallucination_scores'].append(hallucination_score) else: break if hallucination_score > self.react_configuration.hallucination_threshold and result['adherence']==False: - result['response'].append(f"Even after {self.react_configuration.max_attempts} attempts of AIMon react, the LLM neither adheres to the user instructions, nor generates a response that is not hallucinated. Final LLM response: {generated_text}") - result['hscore'].append(hallucination_score) + result['responses'].append(f"Even after {self.react_configuration.max_attempts} attempts of AIMon react, the LLM neither adheres to the user instructions, nor generates a response that is not hallucinated. Final LLM response: {generated_text}") + result['hallucination_scores'].append(hallucination_score) result['react_score']=0 elif hallucination_score > self.react_configuration.hallucination_threshold and result['adherence']==True: - result['response'].append(f"Although the LLM adheres to the user instructions, the generated response, even after {self.react_configuration.max_attempts} attempts of AIMon ReAct is still hallucinated. Final LLM response: {generated_text}") - result['hscore'].append(hallucination_score) + result['responses'].append(f"Although the LLM adheres to the user instructions, the generated response, even after {self.react_configuration.max_attempts} attempts of AIMon ReAct is still hallucinated. Final LLM response: {generated_text}") + result['hallucination_scores'].append(hallucination_score) result['react_score']=0.5 elif hallucination_score <= self.react_configuration.hallucination_threshold and result['adherence']==False: - result['response'].append(f"Although the LLM generates a non-hallucinated response, it fails to adhere to the user instructions, even after {self.react_configuration.max_attempts} attempts of AIMon ReAct. Final LLM response: {generated_text}") - result['hscore'].append(hallucination_score) + result['responses'].append(f"Although the LLM generates a non-hallucinated response, it fails to adhere to the user instructions, even after {self.react_configuration.max_attempts} attempts of AIMon ReAct. Final LLM response: {generated_text}") + result['hallucination_scores'].append(hallucination_score) result['react_score']=0.5 else: - result['response'].append(f"This response is below the hallucination threshold and adheres to the user instructions. Response {generated_text}") - result['hscore'].append(hallucination_score) + result['responses'].append(f"This response is below the hallucination threshold and adheres to the user instructions. Response {generated_text}") + result['hallucination_scores'].append(hallucination_score) result['react_score']=1 return result \ No newline at end of file From abbf007917de7ab36239a059d53db22300e01360 Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Wed, 15 Jan 2025 00:53:24 +0530 Subject: [PATCH 16/24] updated adherence logic --- aimon/extensions/react.py | 49 ++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py index 0b66d79..d045248 100644 --- a/aimon/extensions/react.py +++ b/aimon/extensions/react.py @@ -72,8 +72,8 @@ def react(self, user_query, user_instructions,): result = { 'responses': [], 'hallucination_scores': [], - 'adherence': False, ## False by assumption - 'react_score': 0 ## 0 by assumption + 'adherence': [], + 'react_score': 0 ## 0 by assumption } llm_response = self.llm_app(user_query, user_instructions, reprompted_flag=False) @@ -92,24 +92,23 @@ def react(self, user_query, user_instructions,): hallucination_score = detect_response.hallucination['score'] - # result.append([generated_text, hallucination_score]) + failed_instruction_counter = 0 + ## Loop to check for failed instructions + for x in detect_response.instruction_adherence['results']: + if x['adherence'] == False: + failed_instruction_counter+=1 + adherence = False + break + if failed_instruction_counter == 0: + adherence = True + + ## Baseline + result['adherence'].append(adherence) result['responses'].append(generated_text) result['hallucination_scores'].append(hallucination_score) for _ in range(self.react_configuration.max_attempts): - failed_instructions = [] - failed_instruction_counter = 0 - ## Loop to check for failed instructions - for x in detect_response.instruction_adherence['results']: - if x['adherence'] == False: - failed_instruction_counter+=1 - failed_instructions.append(x['instruction']) - result['adherence']=False - - if failed_instruction_counter == 0: - result['adherence']=True - ## Check whether the hallucination score is greater than the required threshold OR if any of the supplied instructions are not complied with if self.react_configuration.hallucination_threshold > 0 and \ (hallucination_score > self.react_configuration.hallucination_threshold or failed_instruction_counter>0): @@ -130,22 +129,34 @@ def react(self, user_query, user_instructions,): hallucination_score = detect_response.hallucination['score'] + + failed_instruction_counter = 0 + + for x in detect_response.instruction_adherence['results']: + if x['adherence'] == False: + failed_instruction_counter+=1 + adherence = False + break + if failed_instruction_counter == 0: + adherence = True + + result['adherence'].append(adherence) result['responses'].append(generated_text) result['hallucination_scores'].append(hallucination_score) else: break - - if hallucination_score > self.react_configuration.hallucination_threshold and result['adherence']==False: + ## The hallucination score and adherence here are from the last attempt of the max attempts + if hallucination_score > self.react_configuration.hallucination_threshold and adherence==False: result['responses'].append(f"Even after {self.react_configuration.max_attempts} attempts of AIMon react, the LLM neither adheres to the user instructions, nor generates a response that is not hallucinated. Final LLM response: {generated_text}") result['hallucination_scores'].append(hallucination_score) result['react_score']=0 - elif hallucination_score > self.react_configuration.hallucination_threshold and result['adherence']==True: + elif hallucination_score > self.react_configuration.hallucination_threshold and adherence==True: result['responses'].append(f"Although the LLM adheres to the user instructions, the generated response, even after {self.react_configuration.max_attempts} attempts of AIMon ReAct is still hallucinated. Final LLM response: {generated_text}") result['hallucination_scores'].append(hallucination_score) result['react_score']=0.5 - elif hallucination_score <= self.react_configuration.hallucination_threshold and result['adherence']==False: + elif hallucination_score <= self.react_configuration.hallucination_threshold and adherence==False: result['responses'].append(f"Although the LLM generates a non-hallucinated response, it fails to adhere to the user instructions, even after {self.react_configuration.max_attempts} attempts of AIMon ReAct. Final LLM response: {generated_text}") result['hallucination_scores'].append(hallucination_score) result['react_score']=0.5 From e9620373b5657bff7249bf068b45c018faaafae6 Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Wed, 15 Jan 2025 01:33:25 +0530 Subject: [PATCH 17/24] commented some logic that should rather be included in the application notebook --- aimon/extensions/react.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py index d045248..bf89e88 100644 --- a/aimon/extensions/react.py +++ b/aimon/extensions/react.py @@ -149,20 +149,20 @@ def react(self, user_query, user_instructions,): ## The hallucination score and adherence here are from the last attempt of the max attempts if hallucination_score > self.react_configuration.hallucination_threshold and adherence==False: - result['responses'].append(f"Even after {self.react_configuration.max_attempts} attempts of AIMon react, the LLM neither adheres to the user instructions, nor generates a response that is not hallucinated. Final LLM response: {generated_text}") - result['hallucination_scores'].append(hallucination_score) + # result['responses'].append(f"Even after {self.react_configuration.max_attempts} attempts of AIMon react, the LLM neither adheres to the user instructions, nor generates a response that is not hallucinated. Final LLM response: {generated_text}") + # result['hallucination_scores'].append(hallucination_score) result['react_score']=0 elif hallucination_score > self.react_configuration.hallucination_threshold and adherence==True: - result['responses'].append(f"Although the LLM adheres to the user instructions, the generated response, even after {self.react_configuration.max_attempts} attempts of AIMon ReAct is still hallucinated. Final LLM response: {generated_text}") - result['hallucination_scores'].append(hallucination_score) + # result['responses'].append(f"Although the LLM adheres to the user instructions, the generated response, even after {self.react_configuration.max_attempts} attempts of AIMon ReAct is still hallucinated. Final LLM response: {generated_text}") + # result['hallucination_scores'].append(hallucination_score) result['react_score']=0.5 elif hallucination_score <= self.react_configuration.hallucination_threshold and adherence==False: - result['responses'].append(f"Although the LLM generates a non-hallucinated response, it fails to adhere to the user instructions, even after {self.react_configuration.max_attempts} attempts of AIMon ReAct. Final LLM response: {generated_text}") - result['hallucination_scores'].append(hallucination_score) + # result['responses'].append(f"Although the LLM generates a non-hallucinated response, it fails to adhere to the user instructions, even after {self.react_configuration.max_attempts} attempts of AIMon ReAct. Final LLM response: {generated_text}") + # result['hallucination_scores'].append(hallucination_score) result['react_score']=0.5 else: - result['responses'].append(f"This response is below the hallucination threshold and adheres to the user instructions. Response {generated_text}") - result['hallucination_scores'].append(hallucination_score) + # result['responses'].append(f"This response is below the hallucination threshold and adheres to the user instructions. Response {generated_text}") + # result['hallucination_scores'].append(hallucination_score) result['react_score']=1 return result \ No newline at end of file From 705eb5e627634051c67ec1478f387e732a04da16 Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Wed, 15 Jan 2025 05:46:52 +0530 Subject: [PATCH 18/24] resolved feedback --- aimon/extensions/react.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py index bf89e88..a98bc4e 100644 --- a/aimon/extensions/react.py +++ b/aimon/extensions/react.py @@ -1,16 +1,25 @@ +import os from aimon import Client from typing import Optional from dataclasses import dataclass +import random +import string + +def generate_random_string(length): + """Generates a random string of letters and digits.""" + characters = string.ascii_letters + string.digits + return ''.join(random.choice(characters) for i in range(length)) + @dataclass class ReactConfig: publish: bool max_attempts: int - aimon_api_key: str hallucination_threshold: float framework: Optional[str] = None - model_name: Optional[str] = "aimon-react-model" - application_name: Optional[str] = "aimon-react-application" + aimon_api_key: Optional[str] = os.getenv("AIMON_API_KEY") + model_name: Optional[str] = "aimon-react-model" + generate_random_string(5) + application_name: Optional[str] = "aimon-react-application" + generate_random_string(5) class React: @@ -65,15 +74,15 @@ def react(self, user_query, user_instructions,): AIMon-ReAct -> Reason and Act with AIMon where ReAct score: - 1 if after n attempts of ReAct, the LLM follows instructions and generates a non hallucinated response. - 0.5 if after n attempts of ReAct, either the LLM response follows user_instructions or is under hallucination threshold [BUT NOT BOTH]. - 0 otherwise. + 1.0 if after n attempts of ReAct, the LLM follows instructions and generates a non hallucinated response. + 0.5 if after n attempts of ReAct, either the LLM response follows user_instructions or is under hallucination threshold [BUT NOT BOTH]. + 0.0 otherwise. """ result = { 'responses': [], 'hallucination_scores': [], 'adherence': [], - 'react_score': 0 ## 0 by assumption + 'react_score': 0.0 ## 0.0 by assumption } llm_response = self.llm_app(user_query, user_instructions, reprompted_flag=False) @@ -81,7 +90,7 @@ def react(self, user_query, user_instructions,): context = self.context_extractor(user_query, user_instructions, llm_response) ## Generated text for LLM Response, if the user employs the LlamaIndex framework - if llm_response.response or self.react_configuration.framework=="llamaindex": + if self.react_configuration.framework=="llamaindex" or llm_response.response: generated_text = llm_response.response else: generated_text = llm_response @@ -118,7 +127,7 @@ def react(self, user_query, user_instructions,): context = self.context_extractor(user_query, user_instructions, llm_response) ## Generated text for LLM Response, if the user employs the LlamaIndex framework - if llm_response.response or self.react_configuration.framework=="llamaindex": + if self.react_configuration.framework=="llamaindex" or llm_response.response: generated_text = llm_response.response else: generated_text = llm_response @@ -151,7 +160,7 @@ def react(self, user_query, user_instructions,): if hallucination_score > self.react_configuration.hallucination_threshold and adherence==False: # result['responses'].append(f"Even after {self.react_configuration.max_attempts} attempts of AIMon react, the LLM neither adheres to the user instructions, nor generates a response that is not hallucinated. Final LLM response: {generated_text}") # result['hallucination_scores'].append(hallucination_score) - result['react_score']=0 + result['react_score']=0.0 elif hallucination_score > self.react_configuration.hallucination_threshold and adherence==True: # result['responses'].append(f"Although the LLM adheres to the user instructions, the generated response, even after {self.react_configuration.max_attempts} attempts of AIMon ReAct is still hallucinated. Final LLM response: {generated_text}") # result['hallucination_scores'].append(hallucination_score) @@ -163,6 +172,6 @@ def react(self, user_query, user_instructions,): else: # result['responses'].append(f"This response is below the hallucination threshold and adheres to the user instructions. Response {generated_text}") # result['hallucination_scores'].append(hallucination_score) - result['react_score']=1 + result['react_score']=1.0 return result \ No newline at end of file From 3e1d4799ca3d0280afdd130d66d84e1f0c5d75c3 Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Thu, 16 Jan 2025 00:36:27 +0530 Subject: [PATCH 19/24] added support to include IAFeedback and LLMResponse in ReAct runs --- aimon/extensions/react.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py index a98bc4e..ac70d41 100644 --- a/aimon/extensions/react.py +++ b/aimon/extensions/react.py @@ -102,15 +102,17 @@ def react(self, user_query, user_instructions,): hallucination_score = detect_response.hallucination['score'] failed_instruction_counter = 0 + ia_feedback = "" ## Loop to check for failed instructions for x in detect_response.instruction_adherence['results']: if x['adherence'] == False: failed_instruction_counter+=1 adherence = False - break + ia_feedback += f"For the instruction {x['instruction']}, the adherence detector evaluated that {x['detailed_explanation']}." if failed_instruction_counter == 0: adherence = True - + if ia_feedback=="": + ia_feedback="All the instructions were complied with." ## Baseline result['adherence'].append(adherence) result['responses'].append(generated_text) @@ -122,7 +124,12 @@ def react(self, user_query, user_instructions,): if self.react_configuration.hallucination_threshold > 0 and \ (hallucination_score > self.react_configuration.hallucination_threshold or failed_instruction_counter>0): - llm_response = self.llm_app(user_query, user_instructions, reprompted_flag=True, hallucination_score=hallucination_score) + llm_response = self.llm_app(user_query, + user_instructions, + reprompted_flag=True, + last_llm_response = generated_text, + hallucination_score=hallucination_score, + ia_feedback = ia_feedback) context = self.context_extractor(user_query, user_instructions, llm_response) @@ -138,16 +145,18 @@ def react(self, user_query, user_instructions,): hallucination_score = detect_response.hallucination['score'] - + ia_feedback = "" failed_instruction_counter = 0 for x in detect_response.instruction_adherence['results']: if x['adherence'] == False: failed_instruction_counter+=1 adherence = False - break + ia_feedback += f"For the instruction {x['instruction']}, the adherence detector evaluated that {x['detailed_explanation']}." if failed_instruction_counter == 0: adherence = True + if ia_feedback=="": + ia_feedback="All the instructions were complied with." result['adherence'].append(adherence) result['responses'].append(generated_text) From 8e54576d5ead193f4c8c5f0e41db996eef655da5 Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Thu, 30 Jan 2025 14:42:26 -0500 Subject: [PATCH 20/24] added support for predefined response and context --- aimon/extensions/react.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py index ac70d41..cc8001d 100644 --- a/aimon/extensions/react.py +++ b/aimon/extensions/react.py @@ -69,7 +69,7 @@ def detect_aimon_response(self,aimon_payload): return detect_result ## ReAct -> Reason and Act - def react(self, user_query, user_instructions,): + def react(self, user_query, user_instructions, context = None, llm_response = None): """ AIMon-ReAct -> Reason and Act with AIMon @@ -85,12 +85,13 @@ def react(self, user_query, user_instructions,): 'react_score': 0.0 ## 0.0 by assumption } - llm_response = self.llm_app(user_query, user_instructions, reprompted_flag=False) - - context = self.context_extractor(user_query, user_instructions, llm_response) + if llm_response == None: + llm_response = self.llm_app(user_query, user_instructions, reprompted_flag=False) + + if context == None: + context = self.context_extractor(user_query, user_instructions, llm_response) - ## Generated text for LLM Response, if the user employs the LlamaIndex framework - if self.react_configuration.framework=="llamaindex" or llm_response.response: + if llm_response.response: generated_text = llm_response.response else: generated_text = llm_response From 08a241ba6ffadbe8b7f4d496d7eb7f2e5b6ee59d Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Thu, 30 Jan 2025 14:53:50 -0500 Subject: [PATCH 21/24] added isinstance and hasattr conditionals --- aimon/extensions/react.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py index cc8001d..abfe29c 100644 --- a/aimon/extensions/react.py +++ b/aimon/extensions/react.py @@ -91,7 +91,9 @@ def react(self, user_query, user_instructions, context = None, llm_response = No if context == None: context = self.context_extractor(user_query, user_instructions, llm_response) - if llm_response.response: + if isinstance(llm_response, str): + generated_text = llm_response + elif self.react_configuration.framework=="llamaindex" or hasattr(llm_response, 'response'): generated_text = llm_response.response else: generated_text = llm_response From 9635d507d25d2cc1c4c3cd951105df9cfa083566 Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Sat, 1 Feb 2025 06:47:46 -0500 Subject: [PATCH 22/24] Removed support for predefined response and context. It would instead be handled in the LLM app. --- aimon/extensions/react.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py index abfe29c..fbf9a6a 100644 --- a/aimon/extensions/react.py +++ b/aimon/extensions/react.py @@ -69,7 +69,7 @@ def detect_aimon_response(self,aimon_payload): return detect_result ## ReAct -> Reason and Act - def react(self, user_query, user_instructions, context = None, llm_response = None): + def react(self, user_query, user_instructions): """ AIMon-ReAct -> Reason and Act with AIMon @@ -85,11 +85,9 @@ def react(self, user_query, user_instructions, context = None, llm_response = No 'react_score': 0.0 ## 0.0 by assumption } - if llm_response == None: - llm_response = self.llm_app(user_query, user_instructions, reprompted_flag=False) + llm_response = self.llm_app(user_query, user_instructions, reprompted_flag=False) - if context == None: - context = self.context_extractor(user_query, user_instructions, llm_response) + context = self.context_extractor(user_query, user_instructions, llm_response) if isinstance(llm_response, str): generated_text = llm_response From 16111e1aa46eed9db8eb374537ed4b8dd6fba229 Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Tue, 4 Feb 2025 12:18:04 -0500 Subject: [PATCH 23/24] updates based on PR feedback --- aimon/extensions/react.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/aimon/extensions/react.py b/aimon/extensions/react.py index fbf9a6a..00b66b1 100644 --- a/aimon/extensions/react.py +++ b/aimon/extensions/react.py @@ -1,4 +1,5 @@ import os +from enum import Enum from aimon import Client from typing import Optional from dataclasses import dataclass @@ -11,12 +12,34 @@ def generate_random_string(length): characters = string.ascii_letters + string.digits return ''.join(random.choice(characters) for i in range(length)) + +class Framework(Enum): + LLAMAINDEX = "LlamaIndex" + LANGCHAIN = "LangChain" + HAYSTACK = "Haystack" + NONE = None + @dataclass class ReactConfig: + """ + Configuration class for the React configuration settings. + + Attributes: + publish (bool): Flag indicating whether to publish the results to app.aimon.ai + max_attempts (int): Maximum number of ReAct attempts + hallucination_threshold (float): Threshold value to determine hallucination behavior. Defaults to 0.5. + framework (Optional[Framework]): Optional framework configuration. Defaults to None. + aimon_api_key (Optional[str]): API key for AIMon integration. If not provided, it attempts to retrieve from environment variable "AIMON_API_KEY". + model_name (Optional[str]): Name of the model to be used. Defaults to a string based on "aimon-react-model" concatenated with a random string. + application_name (Optional[str]): Name of the application. Defaults to a string based on "aimon-react-application" concatenated with a random string. + + Methods: + None + """ publish: bool max_attempts: int - hallucination_threshold: float - framework: Optional[str] = None + hallucination_threshold: float = 0.5 + framework: Optional[Framework] = None aimon_api_key: Optional[str] = os.getenv("AIMON_API_KEY") model_name: Optional[str] = "aimon-react-model" + generate_random_string(5) application_name: Optional[str] = "aimon-react-application" + generate_random_string(5) @@ -91,7 +114,7 @@ def react(self, user_query, user_instructions): if isinstance(llm_response, str): generated_text = llm_response - elif self.react_configuration.framework=="llamaindex" or hasattr(llm_response, 'response'): + elif self.react_configuration.framework.value=="LlamaIndex" or hasattr(llm_response, 'response'): generated_text = llm_response.response else: generated_text = llm_response @@ -135,7 +158,7 @@ def react(self, user_query, user_instructions): context = self.context_extractor(user_query, user_instructions, llm_response) ## Generated text for LLM Response, if the user employs the LlamaIndex framework - if self.react_configuration.framework=="llamaindex" or llm_response.response: + if self.react_configuration.framework.value=="LlamaIndex" or llm_response.response: generated_text = llm_response.response else: generated_text = llm_response From 632443ce8e836f39f096502d9d846d6653fb4ef0 Mon Sep 17 00:00:00 2001 From: devvratbhardwaj Date: Tue, 4 Feb 2025 12:18:59 -0500 Subject: [PATCH 24/24] init update --- aimon/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aimon/extensions/__init__.py b/aimon/extensions/__init__.py index 99bcce2..fb3a37e 100644 --- a/aimon/extensions/__init__.py +++ b/aimon/extensions/__init__.py @@ -1 +1 @@ -from .react import ReactConfig, React +from .react import ReactConfig, React, Framework