From 57cf98b48412e89897e7ca546bf6afb41843dfc0 Mon Sep 17 00:00:00 2001 From: pliny <133052465+elder-plinius@users.noreply.github.com> Date: Fri, 1 Dec 2023 08:39:33 -0800 Subject: [PATCH 1/6] Update gpt4_vision_api.py --- swarms/models/gpt4_vision_api.py | 101 +++++++++++-------------------- 1 file changed, 34 insertions(+), 67 deletions(-) diff --git a/swarms/models/gpt4_vision_api.py b/swarms/models/gpt4_vision_api.py index 6efb68f4..b589fe70 100644 --- a/swarms/models/gpt4_vision_api.py +++ b/swarms/models/gpt4_vision_api.py @@ -15,10 +15,7 @@ from termcolor import colored try: import cv2 except ImportError: - print( - "OpenCV not installed. Please install OpenCV to use this" - " model." - ) + print("OpenCV not installed. Please install OpenCV to use this model.") raise ImportError # Load environment variables @@ -103,79 +100,60 @@ class GPT4VisionAPI: if self.meta_prompt: self.system_prompt = self.meta_prompt_init() - def encode_image(self, img: str): + def encode_image(self, img_path: str): """Encode image to base64.""" - if not os.path.exists(img): - print(f"Image file not found: {img}") + if not os.path.exists(img_path): + print(f"Image file not found: {img_path}") return None - with open(img, "rb") as image_file: + with open(img_path, "rb") as image_file: return base64.b64encode(image_file.read()).decode("utf-8") + + def download_img_then_encode(self, img: str): """Download image from URL then encode image to base64 using requests""" pass # Function to handle vision tasks def run( - self, - task: Optional[str] = None, - img: Optional[str] = None, - *args, - **kwargs, - ): + self, + image_path, + task): """Run the model.""" try: - base64_image = self.encode_image(img) + base64_image = self.encode_image(image_path) headers = { "Content-Type": "application/json", - "Authorization": f"Bearer {openai_api_key}", + "Authorization": f"Bearer {self.openai_api_key}", } payload = { "model": self.model_name, "messages": [ - { - "role": "system", - "content": [self.system_prompt], - }, + {"role": "system", "content": [self.system_prompt]}, { "role": "user", "content": [ {"type": "text", "text": task}, - { - "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{base64_image}" - }, - }, + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}} ], }, ], "max_tokens": self.max_tokens, } - response = requests.post( - self.openai_proxy, - headers=headers, - json=payload, - ) + response = requests.post(self.openai_proxy, headers=headers, json=payload) out = response.json() - content = out["choices"][0]["message"]["content"] - - if self.streaming_enabled: - content = self.stream_response(content) - else: - pass - - if self.beautify: - content = colored(content, "cyan") - print(content) + if 'choices' in out and out['choices']: + content = out["choices"][0].get("message", {}).get("content", None) + return content else: - print(content) + print("No valid response in 'choices'") + return None except Exception as error: print(f"Error with the request: {error}") - raise error + return None def video_prompt(self, frames): """ @@ -249,9 +227,7 @@ class GPT4VisionAPI: if not success: break _, buffer = cv2.imencode(".jpg", frame) - base64_frames.append( - base64.b64encode(buffer).decode("utf-8") - ) + base64_frames.append(base64.b64encode(buffer).decode("utf-8")) video.release() print(len(base64_frames), "frames read.") @@ -276,10 +252,7 @@ class GPT4VisionAPI: payload = { "model": self.model_name, "messages": [ - { - "role": "system", - "content": [self.system_prompt], - }, + {"role": "system", "content": [self.system_prompt]}, { "role": "user", "content": [ @@ -287,7 +260,9 @@ class GPT4VisionAPI: { "type": "image_url", "image_url": { - "url": f"data:image/jpeg;base64,{base64_image}" + "url": ( + f"data:image/jpeg;base64,{base64_image}" + ) }, }, ], @@ -337,9 +312,7 @@ class GPT4VisionAPI: """ # Instantiate the thread pool executor - with ThreadPoolExecutor( - max_workers=self.max_workers - ) as executor: + with ThreadPoolExecutor(max_workers=self.max_workers) as executor: results = executor.map(self.run, tasks, imgs) # Print the results for debugging @@ -385,7 +358,9 @@ class GPT4VisionAPI: { "type": "image_url", "image_url": { - "url": f"data:image/jpeg;base64,{base64_image}" + "url": ( + f"data:image/jpeg;base64,{base64_image}" + ) }, }, ], @@ -395,9 +370,7 @@ class GPT4VisionAPI: } async with aiohttp.ClientSession() as session: async with session.post( - self.openai_proxy, - headers=headers, - data=json.dumps(payload), + self.openai_proxy, headers=headers, data=json.dumps(payload) ) as response: out = await response.json() content = out["choices"][0]["message"]["content"] @@ -406,9 +379,7 @@ class GPT4VisionAPI: print(f"Error with the request {error}") raise error - def run_batch( - self, tasks_images: List[Tuple[str, str]] - ) -> List[str]: + def run_batch(self, tasks_images: List[Tuple[str, str]]) -> List[str]: """Process a batch of tasks and images""" with concurrent.futures.ThreadPoolExecutor() as executor: futures = [ @@ -435,9 +406,7 @@ class GPT4VisionAPI: """Process a batch of tasks and images asynchronously with retries""" loop = asyncio.get_event_loop() futures = [ - loop.run_in_executor( - None, self.run_with_retries, task, img - ) + loop.run_in_executor(None, self.run_with_retries, task, img) for task, img in tasks_images ] return await asyncio.gather(*futures) @@ -445,9 +414,7 @@ class GPT4VisionAPI: def health_check(self): """Health check for the GPT4Vision model""" try: - response = requests.get( - "https://api.openai.com/v1/engines" - ) + response = requests.get("https://api.openai.com/v1/engines") return response.status_code == 200 except requests.RequestException as error: print(f"Health check failed: {error}") From 6fa37ef40aaba71a62a44846d6231c5d89593095 Mon Sep 17 00:00:00 2001 From: pliny <133052465+elder-plinius@users.noreply.github.com> Date: Sat, 2 Dec 2023 13:35:03 -0800 Subject: [PATCH 2/6] Create education.py --- playground/demos/education/education.py | 56 +++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 playground/demos/education/education.py diff --git a/playground/demos/education/education.py b/playground/demos/education/education.py new file mode 100644 index 00000000..5b9d9df7 --- /dev/null +++ b/playground/demos/education/education.py @@ -0,0 +1,56 @@ +import os +from dotenv import load_dotenv +from swarms.models import OpenAIChat +from swarms.models.stable_diffusion import StableDiffusion +from swarms.structs import Agent, SequentialWorkflow +import swarms.prompts.education as edu_prompts + +# Load environment variables +load_dotenv() +api_key = os.getenv("OPENAI_API_KEY") +stability_api_key = os.getenv("STABILITY_API_KEY") + +# Initialize language model +llm = OpenAIChat(openai_api_key=api_key, temperature=0.5, max_tokens=3000) + +# Initialize Stable Diffusion +sd_api = StableDiffusion(api_key=stability_api_key) + +# User preferences (can be dynamically set in a real application) +user_preferences = { + "subjects": "Cognitive Architectures", + "learning_style": "Visual", + "challenge_level": "Moderate" +} + +# Formatted prompts from user preferences +curriculum_prompt = edu_prompts.CURRICULUM_DESIGN_PROMPT.format(**user_preferences) +interactive_prompt = edu_prompts.INTERACTIVE_LEARNING_PROMPT.format(**user_preferences) +sample_prompt = edu_prompts.SAMPLE_TEST_PROMPT.format(**user_preferences) +image_prompt = edu_prompts.IMAGE_GENERATION_PROMPT.format(**user_preferences) + +# Initialize agents for different educational tasks +curriculum_agent = Agent(llm=llm, max_loops=1, sop=curriculum_prompt) +interactive_learning_agent = Agent(llm=llm, max_loops=1, sop=interactive_prompt) +sample_lesson_agent = Agent(llm=llm, max_loops=1, sop=sample_prompt) + +# Create Sequential Workflow +workflow = SequentialWorkflow(max_loops=1) + +# Add tasks to workflow with personalized prompts +workflow.add(curriculum_agent, "Generate a curriculum") +workflow.add(interactive_learning_agent, "Generate an interactive lesson") +workflow.add(sample_lesson_agent, "Generate a practice test") + +# Execute the workflow for text-based tasks +workflow.run() + +# Generate an image using Stable Diffusion +image_result = sd_api.run(image_prompt) + +# Output results for each task +for task in workflow.tasks: + print(f"Task Description: {task.description}\nResult: {task.result}\n") + +# Output image result +print(f"Image Generation Task: Generate an image for the interactive lesson\nResult: {image_result}") From 74bb90671df583967f613f40e75c326f13b0d174 Mon Sep 17 00:00:00 2001 From: pliny <133052465+elder-plinius@users.noreply.github.com> Date: Sat, 2 Dec 2023 13:37:21 -0800 Subject: [PATCH 3/6] Create education.py --- swarms/prompts/education.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 swarms/prompts/education.py diff --git a/swarms/prompts/education.py b/swarms/prompts/education.py new file mode 100644 index 00000000..5016c0f8 --- /dev/null +++ b/swarms/prompts/education.py @@ -0,0 +1,37 @@ +user_preferences = { + "subjects": "AI Cognitive Architectures", + "learning_style": "Visual", + "challenge_level": "Moderate", +} + +# Extracting individual preferences +subjects = user_preferences["subjects"] +learning_style = user_preferences["learning_style"] +challenge_level = user_preferences["challenge_level"] + + +# Curriculum Design Prompt +CURRICULUM_DESIGN_PROMPT = f""" +Develop a semester-long curriculum tailored to student interests in {subjects}. Focus on incorporating diverse teaching methods suitable for a {learning_style} learning style. +The curriculum should challenge students at a {challenge_level} level, integrating both theoretical knowledge and practical applications. Provide a detailed structure, including weekly topics, +key objectives, and essential resources needed. +""" + +# Interactive Learning Session Prompt +INTERACTIVE_LEARNING_PROMPT = f""" +You are being given a curriculum from another agent, please design an interactive learning sessions for the {subjects} curriculum that cater to a {learning_style} learning style. Incorporate +engaging elements like gamification, interactive quizzes, and hands-on activities. The sessions should dynamically adjust to the student's engagement and understanding, emphasizing key concepts +and practical skills. +""" + +# Sample Lesson Prompt +SAMPLE_TEST_PROMPT = f""" +Create a comprehensive sample test for the first week of the {subjects} curriculum, tailored for a {learning_style} learning style and at a {challenge_level} challenge level. +""" + +# Image Generation for Education Prompt +IMAGE_GENERATION_PROMPT = f""" +Develop a stable diffusion prompt for an educational image/visual aid that align with the {subjects} curriculum, specifically designed to enhance understanding for students with a {learning_style} +learning style. This might include diagrams, infographics, and illustrative representations to simplify complex concepts. Ensure you output a 10/10 descriptive image generation prompt only. +""" + From 870530b9223dd83dcab2dcd0bcc9a7e8b5018624 Mon Sep 17 00:00:00 2001 From: pliny <133052465+elder-plinius@users.noreply.github.com> Date: Sat, 2 Dec 2023 13:48:13 -0800 Subject: [PATCH 4/6] Update education.py --- swarms/prompts/education.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/swarms/prompts/education.py b/swarms/prompts/education.py index 5016c0f8..aa41716b 100644 --- a/swarms/prompts/education.py +++ b/swarms/prompts/education.py @@ -13,15 +13,13 @@ challenge_level = user_preferences["challenge_level"] # Curriculum Design Prompt CURRICULUM_DESIGN_PROMPT = f""" Develop a semester-long curriculum tailored to student interests in {subjects}. Focus on incorporating diverse teaching methods suitable for a {learning_style} learning style. -The curriculum should challenge students at a {challenge_level} level, integrating both theoretical knowledge and practical applications. Provide a detailed structure, including weekly topics, -key objectives, and essential resources needed. +The curriculum should challenge students at a {challenge_level} level, integrating both theoretical knowledge and practical applications. Provide a detailed structure, including +weekly topics, key objectives, and essential resources needed. """ # Interactive Learning Session Prompt INTERACTIVE_LEARNING_PROMPT = f""" -You are being given a curriculum from another agent, please design an interactive learning sessions for the {subjects} curriculum that cater to a {learning_style} learning style. Incorporate -engaging elements like gamification, interactive quizzes, and hands-on activities. The sessions should dynamically adjust to the student's engagement and understanding, emphasizing key concepts -and practical skills. +Create an interactive lesson for a student of {subjects} that caters to a {learning_style} learning style. Incorporate engaging elements and hands-on activities. """ # Sample Lesson Prompt From f50907ab3ed496439e6e112bc7d147d882ed5c5d Mon Sep 17 00:00:00 2001 From: pliny <133052465+elder-plinius@users.noreply.github.com> Date: Sat, 2 Dec 2023 13:48:57 -0800 Subject: [PATCH 5/6] Update education.py --- swarms/prompts/education.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarms/prompts/education.py b/swarms/prompts/education.py index aa41716b..0e7742d9 100644 --- a/swarms/prompts/education.py +++ b/swarms/prompts/education.py @@ -19,7 +19,7 @@ weekly topics, key objectives, and essential resources needed. # Interactive Learning Session Prompt INTERACTIVE_LEARNING_PROMPT = f""" -Create an interactive lesson for a student of {subjects} that caters to a {learning_style} learning style. Incorporate engaging elements and hands-on activities. +Basedon the curriculum, generate an interactive lesson plan for a student of {subjects} that caters to a {learning_style} learning style. Incorporate engaging elements and hands-on activities. """ # Sample Lesson Prompt From 090a3687285bd24ff445517314fd97ec5dd359d9 Mon Sep 17 00:00:00 2001 From: pliny <133052465+elder-plinius@users.noreply.github.com> Date: Sat, 2 Dec 2023 15:12:34 -0800 Subject: [PATCH 6/6] Update gpt4_vision_api.py --- swarms/models/gpt4_vision_api.py | 83 ++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 30 deletions(-) diff --git a/swarms/models/gpt4_vision_api.py b/swarms/models/gpt4_vision_api.py index b589fe70..cd6e5ddb 100644 --- a/swarms/models/gpt4_vision_api.py +++ b/swarms/models/gpt4_vision_api.py @@ -15,7 +15,10 @@ from termcolor import colored try: import cv2 except ImportError: - print("OpenCV not installed. Please install OpenCV to use this model.") + print( + "OpenCV not installed. Please install OpenCV to use this" + " model." + ) raise ImportError # Load environment variables @@ -100,29 +103,24 @@ class GPT4VisionAPI: if self.meta_prompt: self.system_prompt = self.meta_prompt_init() - def encode_image(self, img_path: str): + def encode_image(self, img: str): """Encode image to base64.""" - if not os.path.exists(img_path): - print(f"Image file not found: {img_path}") + if not os.path.exists(img): + print(f"Image file not found: {img}") return None - with open(img_path, "rb") as image_file: + with open(img, "rb") as image_file: return base64.b64encode(image_file.read()).decode("utf-8") - - def download_img_then_encode(self, img: str): """Download image from URL then encode image to base64 using requests""" pass # Function to handle vision tasks - def run( - self, - image_path, - task): + def run(self, img, task): """Run the model.""" try: - base64_image = self.encode_image(image_path) + base64_image = self.encode_image(img) headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self.openai_api_key}", @@ -130,22 +128,36 @@ class GPT4VisionAPI: payload = { "model": self.model_name, "messages": [ - {"role": "system", "content": [self.system_prompt]}, + { + "role": "system", + "content": [self.system_prompt], + }, { "role": "user", "content": [ {"type": "text", "text": task}, - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}} + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{base64_image}" + }, + }, ], }, ], "max_tokens": self.max_tokens, } - response = requests.post(self.openai_proxy, headers=headers, json=payload) + response = requests.post( + self.openai_proxy, headers=headers, json=payload + ) out = response.json() - if 'choices' in out and out['choices']: - content = out["choices"][0].get("message", {}).get("content", None) + if "choices" in out and out["choices"]: + content = ( + out["choices"][0] + .get("message", {}) + .get("content", None) + ) return content else: print("No valid response in 'choices'") @@ -227,7 +239,9 @@ class GPT4VisionAPI: if not success: break _, buffer = cv2.imencode(".jpg", frame) - base64_frames.append(base64.b64encode(buffer).decode("utf-8")) + base64_frames.append( + base64.b64encode(buffer).decode("utf-8") + ) video.release() print(len(base64_frames), "frames read.") @@ -252,7 +266,10 @@ class GPT4VisionAPI: payload = { "model": self.model_name, "messages": [ - {"role": "system", "content": [self.system_prompt]}, + { + "role": "system", + "content": [self.system_prompt], + }, { "role": "user", "content": [ @@ -260,9 +277,7 @@ class GPT4VisionAPI: { "type": "image_url", "image_url": { - "url": ( - f"data:image/jpeg;base64,{base64_image}" - ) + "url": f"data:image/jpeg;base64,{base64_image}" }, }, ], @@ -312,7 +327,9 @@ class GPT4VisionAPI: """ # Instantiate the thread pool executor - with ThreadPoolExecutor(max_workers=self.max_workers) as executor: + with ThreadPoolExecutor( + max_workers=self.max_workers + ) as executor: results = executor.map(self.run, tasks, imgs) # Print the results for debugging @@ -358,9 +375,7 @@ class GPT4VisionAPI: { "type": "image_url", "image_url": { - "url": ( - f"data:image/jpeg;base64,{base64_image}" - ) + "url": f"data:image/jpeg;base64,{base64_image}" }, }, ], @@ -370,7 +385,9 @@ class GPT4VisionAPI: } async with aiohttp.ClientSession() as session: async with session.post( - self.openai_proxy, headers=headers, data=json.dumps(payload) + self.openai_proxy, + headers=headers, + data=json.dumps(payload), ) as response: out = await response.json() content = out["choices"][0]["message"]["content"] @@ -379,7 +396,9 @@ class GPT4VisionAPI: print(f"Error with the request {error}") raise error - def run_batch(self, tasks_images: List[Tuple[str, str]]) -> List[str]: + def run_batch( + self, tasks_images: List[Tuple[str, str]] + ) -> List[str]: """Process a batch of tasks and images""" with concurrent.futures.ThreadPoolExecutor() as executor: futures = [ @@ -406,7 +425,9 @@ class GPT4VisionAPI: """Process a batch of tasks and images asynchronously with retries""" loop = asyncio.get_event_loop() futures = [ - loop.run_in_executor(None, self.run_with_retries, task, img) + loop.run_in_executor( + None, self.run_with_retries, task, img + ) for task, img in tasks_images ] return await asyncio.gather(*futures) @@ -414,7 +435,9 @@ class GPT4VisionAPI: def health_check(self): """Health check for the GPT4Vision model""" try: - response = requests.get("https://api.openai.com/v1/engines") + response = requests.get( + "https://api.openai.com/v1/engines" + ) return response.status_code == 200 except requests.RequestException as error: print(f"Health check failed: {error}")