From 825b66b44b42b873fbf7f06b5d7a295c39722096 Mon Sep 17 00:00:00 2001 From: Kye Date: Wed, 18 Oct 2023 16:15:47 -0400 Subject: [PATCH] Fuyu Former-commit-id: 37d63776c9a35332983ec4d782cb3239220e990e --- swarms/__init__.py | 3 +- swarms/agents/__init__.py | 2 +- swarms/embeddings/openai.py | 6 +++- swarms/memory/__init__.py | 2 +- swarms/models/__init__.py | 1 + swarms/models/fuyu.py | 65 +++++++++++++++++++++++++++++++++++++ swarms/prompts/__init__.py | 2 +- swarms/structs/__init__.py | 2 +- swarms/swarms/__init__.py | 2 +- swarms/utils/__init__.py | 3 +- swarms/workers/__init__.py | 2 +- tests/memory/pinecone.py | 6 ++-- 12 files changed, 82 insertions(+), 14 deletions(-) create mode 100644 swarms/models/fuyu.py diff --git a/swarms/__init__.py b/swarms/__init__.py index df576685..39f960ee 100644 --- a/swarms/__init__.py +++ b/swarms/__init__.py @@ -1,4 +1,3 @@ - import warnings warnings.filterwarnings("ignore", category=UserWarning) @@ -9,8 +8,8 @@ import os os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" - from swarms.logo import logo + print(logo) from swarms import agents diff --git a/swarms/agents/__init__.py b/swarms/agents/__init__.py index fb834232..7c087082 100644 --- a/swarms/agents/__init__.py +++ b/swarms/agents/__init__.py @@ -9,4 +9,4 @@ from swarms.agents.message import Message from swarms.agents.stream_response import stream from swarms.agents.base import AbstractAgent from swarms.agents.registry import Registry -from swarms.agents.idea_to_image_agent import Idea2Image \ No newline at end of file +from swarms.agents.idea_to_image_agent import Idea2Image diff --git a/swarms/embeddings/openai.py b/swarms/embeddings/openai.py index 70327a6c..230dade9 100644 --- a/swarms/embeddings/openai.py +++ b/swarms/embeddings/openai.py @@ -380,7 +380,11 @@ class OpenAIEmbeddings(BaseModel, Embeddings): for i in range(len(texts)): _result = results[i] if len(_result) == 0: - average = embed_with_retry(self, input="", **self._invocation_params,)[ + average = embed_with_retry( + self, + input="", + **self._invocation_params, + )[ "data" ][0]["embedding"] else: diff --git a/swarms/memory/__init__.py b/swarms/memory/__init__.py index 513aa283..dccc5965 100644 --- a/swarms/memory/__init__.py +++ b/swarms/memory/__init__.py @@ -1,4 +1,4 @@ from swarms.memory.vector_stores.pinecone import PineconeVector from swarms.memory.vector_stores.base import BaseVectorStore from swarms.memory.vector_stores.pg import PgVectorVectorStore -from swarms.memory.ocean import OceanDB \ No newline at end of file +from swarms.memory.ocean import OceanDB diff --git a/swarms/models/__init__.py b/swarms/models/__init__.py index 2fdfda1e..100d2543 100644 --- a/swarms/models/__init__.py +++ b/swarms/models/__init__.py @@ -2,3 +2,4 @@ from swarms.models.anthropic import Anthropic from swarms.models.petals import Petals from swarms.models.mistral import Mistral from swarms.models.openai_models import OpenAI, AzureOpenAI, OpenAIChat +from swarms.models.fuyu import Fuyu \ No newline at end of file diff --git a/swarms/models/fuyu.py b/swarms/models/fuyu.py new file mode 100644 index 00000000..4d686b76 --- /dev/null +++ b/swarms/models/fuyu.py @@ -0,0 +1,65 @@ +"""Fuyu model by Kye""" +from transformers import ( + FuyuForCausalLM, + AutoTokenizer, + FuyuProcessor, + FuyuImageProcessor, +) +from PIL import Image + + +class Fuyu: + """ + Fuyu model by Adept + + + Parameters + ---------- + pretrained_path : str + Path to the pretrained model + device_map : str + Device to use for the model + max_new_tokens : int + Maximum number of tokens to generate + + Examples + -------- + >>> fuyu = Fuyu() + >>> fuyu("Hello, my name is", "path/to/image.png") + + + + """ + def __init__( + self, + pretrained_path: str = "adept/fuyu-8b", + device_map: str = "cuda", + max_new_tokens: int = 7, + ): + self.pretrained_path = pretrained_path + self.device_map = device_map + self.max_new_tokens = max_new_tokens + + self.tokenizer = AutoTokenizer.from_pretrained(pretrained_path) + self.image_processor = FuyuImageProcessor() + self.processor = FuyuProcessor( + image_procesor=self.image_processor, tokenizer=self.tokenizer + ) + self.model = FuyuForCausalLM.from_pretrained( + pretrained_path, device_map=device_map + ) + + def __call__(self, text, img_path): + """Call the model""" + image_pil = Image.open(img_path) + model_inputs = self.processor( + text=text, images=[image_pil], device=self.device_map + ) + + for k, v in model_inputs.items(): + model_inputs[k] = v.to(self.device_map) + + output = self.model.generate( + **model_inputs, max_new_tokens=self.fmax_new_tokens + ) + text = self.processor.batch_decode(output[:, -7:], skip_special_tokens=True) diff --git a/swarms/prompts/__init__.py b/swarms/prompts/__init__.py index 24f69d8c..3b873a9d 100644 --- a/swarms/prompts/__init__.py +++ b/swarms/prompts/__init__.py @@ -1 +1 @@ -from swarms.prompts.code_interpreter import CODE_INTERPRETER \ No newline at end of file +from swarms.prompts.code_interpreter import CODE_INTERPRETER diff --git a/swarms/structs/__init__.py b/swarms/structs/__init__.py index 1bd6b86f..c7454474 100644 --- a/swarms/structs/__init__.py +++ b/swarms/structs/__init__.py @@ -1,2 +1,2 @@ from swarms.structs.workflow import Workflow -from swarms.structs.task import Task \ No newline at end of file +from swarms.structs.task import Task diff --git a/swarms/swarms/__init__.py b/swarms/swarms/__init__.py index 4d58d1cd..79d2048b 100644 --- a/swarms/swarms/__init__.py +++ b/swarms/swarms/__init__.py @@ -4,4 +4,4 @@ from swarms.swarms.orchestrate import Orchestrator from swarms.swarms.god_mode import GodMode from swarms.swarms.simple_swarm import SimpleSwarm from swarms.swarms.multi_agent_debate import MultiAgentDebate, select_speaker -from swarms.swarms.groupchat import GroupChat, GroupChatManager \ No newline at end of file +from swarms.swarms.groupchat import GroupChat, GroupChatManager diff --git a/swarms/utils/__init__.py b/swarms/utils/__init__.py index ff014bc0..0b78456a 100644 --- a/swarms/utils/__init__.py +++ b/swarms/utils/__init__.py @@ -1,3 +1,2 @@ - from swarms.utils.display_markdown import display_markdown_message -from swarms.utils.futures import execute_futures_dict \ No newline at end of file +from swarms.utils.futures import execute_futures_dict diff --git a/swarms/workers/__init__.py b/swarms/workers/__init__.py index 67fbf38f..2a7cc4f1 100644 --- a/swarms/workers/__init__.py +++ b/swarms/workers/__init__.py @@ -1,2 +1,2 @@ from swarms.workers.worker import Worker -from swarms.workers.base import AbstractWorker \ No newline at end of file +from swarms.workers.base import AbstractWorker diff --git a/tests/memory/pinecone.py b/tests/memory/pinecone.py index d6c1b76c..bd037bef 100644 --- a/tests/memory/pinecone.py +++ b/tests/memory/pinecone.py @@ -54,9 +54,9 @@ def test_query(): def test_create_index(): - with patch("pinecone.init"), patch( - "pinecone.Index" - ), patch("pinecone.create_index") as MockCreateIndex: + with patch("pinecone.init"), patch("pinecone.Index"), patch( + "pinecone.create_index" + ) as MockCreateIndex: store = PineconeVectorStore( api_key=api_key, index_name="test_index", environment="test_env" )