From 79ef849de158e400aead392d2749214e0ea5b15c Mon Sep 17 00:00:00 2001 From: Sambhav Dixit <94298612+sambhavnoobcoder@users.noreply.github.com> Date: Sat, 19 Oct 2024 00:07:31 +0530 Subject: [PATCH 1/7] Ensure tool execution regardless of long-term memory usage Modified the `run` method in the Agent class to check and execute tools after generating a response, regardless of whether long-term memory is used or not. This fixes the issue where tools were not being executed when long-term memory was present. Changes: - Moved tool execution check and call outside of the long-term memory conditional block - Ensures consistent tool usage across all agent runs --- swarms/structs/agent.py | 60 +++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/swarms/structs/agent.py b/swarms/structs/agent.py index be02c704..9f3692dd 100644 --- a/swarms/structs/agent.py +++ b/swarms/structs/agent.py @@ -711,45 +711,41 @@ class Agent: while attempt < self.retry_attempts and not success: try: if self.long_term_memory is not None: - logger.info( - "Querying long term memory..." - ) + logger.info("Querying long term memory...") self.memory_query(task_prompt) - else: - response_args = ( - (task_prompt, *args) - if img is None - else (task_prompt, img, *args) - ) - response = self.call_llm( - *response_args, **kwargs - ) + # Generate response using LLM + response = self.llm(task_prompt, *args, **kwargs) - # Log the step metadata - logged = self.log_step_metadata( - loop_count, task_prompt, response - ) - logger.info(logged) + # Check and execute tools + if self.tools is not None: + print(f"self.tools is not None: {response}") + self.parse_and_execute_tools(response) - # Conver to a str if the response is not a str - response = self.llm_output_parser( - response - ) + # Log the step metadata + logged = self.log_step_metadata( + loop_count, task_prompt, response + ) + logger.info(logged) - # Print - if self.streaming_on is True: - self.stream_response(response) - else: - print(response) + # Conver to a str if the response is not a str + response = self.llm_output_parser( + response + ) - # Add the response to the memory - self.short_memory.add( - role=self.agent_name, content=response - ) + # Print + if self.streaming_on is True: + self.stream_response(response) + else: + print(response) + + # Add the response to the memory + self.short_memory.add( + role=self.agent_name, content=response + ) - # Add to all responses - all_responses.append(response) + # Add to all responses + all_responses.append(response) # TODO: Implement reliablity check if self.tools is not None: From c60789eb54067eea70cde8d67ed33782df7cb090 Mon Sep 17 00:00:00 2001 From: Sambhav Dixit <94298612+sambhavnoobcoder@users.noreply.github.com> Date: Fri, 25 Oct 2024 02:49:42 +0530 Subject: [PATCH 2/7] response_args should include image - other functionality remains same , code is just slightly rearranged if anything . --- swarms/structs/agent.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/swarms/structs/agent.py b/swarms/structs/agent.py index 5bb9e9b0..29d7037b 100644 --- a/swarms/structs/agent.py +++ b/swarms/structs/agent.py @@ -803,7 +803,14 @@ class Agent: self.memory_query(task_prompt) # Generate response using LLM - response = self.llm(task_prompt, *args, **kwargs) + response_args = ( + (task_prompt, *args) + if img is None + else (task_prompt, img, *args) + ) + response = self.call_llm( + *response_args, **kwargs + ) # Check and execute tools if self.tools is not None: From e06e898486304ce72a53b5e783ab48c3c4db0b34 Mon Sep 17 00:00:00 2001 From: Sambhav Dixit <94298612+sambhavnoobcoder@users.noreply.github.com> Date: Fri, 25 Oct 2024 02:58:47 +0530 Subject: [PATCH 3/7] add section for 'choice' based responses . - when response is a choice , this determines how it is handled. - current implementation uses a placeholder for llm_output_parser , it needs to be updated ( next commit ) --- swarms/structs/agent.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/swarms/structs/agent.py b/swarms/structs/agent.py index 29d7037b..49da5987 100644 --- a/swarms/structs/agent.py +++ b/swarms/structs/agent.py @@ -812,6 +812,15 @@ class Agent: *response_args, **kwargs ) + # Check if response is a dictionary and has 'choices' key + if isinstance(response, dict) and 'choices' in response: + response = response['choices'][0]['message']['content'] + elif isinstance(response, str): + # If response is already a string, use it as is + pass + else: + raise ValueError(f"Unexpected response format: {type(response)}") + # Check and execute tools if self.tools is not None: print(f"self.tools is not None: {response}") From a299a5854e40168656291af03e8db64e72b2e45e Mon Sep 17 00:00:00 2001 From: Sambhav Dixit <94298612+sambhavnoobcoder@users.noreply.github.com> Date: Fri, 25 Oct 2024 03:00:34 +0530 Subject: [PATCH 4/7] proper implemenation of llm_output_parser - implements the decision making , when 'choice' type output is received . --- swarms/structs/agent.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/swarms/structs/agent.py b/swarms/structs/agent.py index 49da5987..36082b61 100644 --- a/swarms/structs/agent.py +++ b/swarms/structs/agent.py @@ -1858,20 +1858,21 @@ class Agent: return response - def llm_output_parser(self, response: Any) -> str: - """ - Parses the response from the LLM (Low-Level Monitor) and returns it as a string. - - Args: - response (Any): The response from the LLM. - - Returns: - str: The parsed response as a string. - """ - if response is not str: - response = str(response) - - return response + def llm_output_parser(self, response): + """Parse the output from the LLM""" + try: + if isinstance(response, dict): + if 'choices' in response: + return response['choices'][0]['message']['content'] + else: + return json.dumps(response) # Convert dict to string + elif isinstance(response, str): + return response + else: + return str(response) # Convert any other type to string + except Exception as e: + logger.error(f"Error parsing LLM output: {e}") + return str(response) # Return string representation as fallback def log_step_metadata( self, loop: int, task: str, response: str From db821f0bd7778fbfe30e2cd7c723307706a0c003 Mon Sep 17 00:00:00 2001 From: Sambhav Dixit <94298612+sambhavnoobcoder@users.noreply.github.com> Date: Fri, 25 Oct 2024 03:18:00 +0530 Subject: [PATCH 5/7] minor refactor --- swarms/structs/agent.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/swarms/structs/agent.py b/swarms/structs/agent.py index 36082b61..a089aee5 100644 --- a/swarms/structs/agent.py +++ b/swarms/structs/agent.py @@ -804,10 +804,10 @@ class Agent: # Generate response using LLM response_args = ( - (task_prompt, *args) - if img is None - else (task_prompt, img, *args) - ) + (task_prompt, *args) + if img is None + else (task_prompt, img, *args) + ) response = self.call_llm( *response_args, **kwargs ) From a0ab6af42ed7d148a68c947035995dd6fa77cfc4 Mon Sep 17 00:00:00 2001 From: Sambhav Dixit <94298612+sambhavnoobcoder@users.noreply.github.com> Date: Fri, 25 Oct 2024 03:37:01 +0530 Subject: [PATCH 6/7] style check --- swarms/structs/agent.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/swarms/structs/agent.py b/swarms/structs/agent.py index a089aee5..e6ca4405 100644 --- a/swarms/structs/agent.py +++ b/swarms/structs/agent.py @@ -803,14 +803,8 @@ class Agent: self.memory_query(task_prompt) # Generate response using LLM - response_args = ( - (task_prompt, *args) - if img is None - else (task_prompt, img, *args) - ) - response = self.call_llm( - *response_args, **kwargs - ) + response = self.llm(task_prompt, *args, **kwargs) + # Check if response is a dictionary and has 'choices' key if isinstance(response, dict) and 'choices' in response: From 486a3cf0f66b0a0a6cbcd57ea2b3a0a26bb53a06 Mon Sep 17 00:00:00 2001 From: Sambhav Dixit <94298612+sambhavnoobcoder@users.noreply.github.com> Date: Fri, 25 Oct 2024 03:50:43 +0530 Subject: [PATCH 7/7] changes --- swarms/structs/agent.py | 44 ++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/swarms/structs/agent.py b/swarms/structs/agent.py index e6ca4405..30073fab 100644 --- a/swarms/structs/agent.py +++ b/swarms/structs/agent.py @@ -801,11 +801,13 @@ class Agent: if self.long_term_memory is not None: logger.info("Querying long term memory...") self.memory_query(task_prompt) - + # Generate response using LLM - response = self.llm(task_prompt, *args, **kwargs) - - + response_args = ( + (task_prompt, *args) if img is None else (task_prompt, img, *args) + ) + response = self.call_llm(*response_args, **kwargs) + # Check if response is a dictionary and has 'choices' key if isinstance(response, dict) and 'choices' in response: response = response['choices'][0]['message']['content'] @@ -813,43 +815,45 @@ class Agent: # If response is already a string, use it as is pass else: - raise ValueError(f"Unexpected response format: {type(response)}") - + raise ValueError( + f"Unexpected response format: {type(response)}" + ) + # Check and execute tools if self.tools is not None: print(f"self.tools is not None: {response}") self.parse_and_execute_tools(response) - + # Log the step metadata logged = self.log_step_metadata( - loop_count, task_prompt, response - ) - logger.info(logged) - - # Conver to a str if the response is not a str - response = self.llm_output_parser( + loop_count, + task_prompt, response ) - + logger.info(logged) + + # Convert to a str if the response is not a str + response = self.llm_output_parser(response) + # Print if self.streaming_on is True: self.stream_response(response) else: print(response) - + # Add the response to the memory self.short_memory.add( - role=self.agent_name, content=response + role=self.agent_name, + content=response ) - + # Add to all responses all_responses.append(response) - - # TODO: Implement reliablity check + + # TODO: Implement reliability check if self.tools is not None: # self.parse_function_call_and_execute(response) self.parse_and_execute_tools(response) - # if self.code_interpreter is True: # # Parse the code and execute # logger.info("Parsing code and executing...")