Fuyu

Former-commit-id: 37d63776c9

swarms/__init__.py

@@ -1,4 +1,3 @@
-
 import warnings
 
 warnings.filterwarnings("ignore", category=UserWarning)
@@ -9,8 +8,8 @@ import os
 os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
 
 
-
 from swarms.logo import logo
+
 print(logo)
 
 from swarms import agents

swarms/embeddings/openai.py

@@ -380,7 +380,11 @@ class OpenAIEmbeddings(BaseModel, Embeddings):
         for i in range(len(texts)):
             _result = results[i]
             if len(_result) == 0:
-                average = embed_with_retry(self, input="", **self._invocation_params,)[
+                average = embed_with_retry(
+                    self,
+                    input="",
+                    **self._invocation_params,
+                )[
                     "data"
                 ][0]["embedding"]
             else:

swarms/models/__init__.py

@@ -2,3 +2,4 @@ from swarms.models.anthropic import Anthropic
 from swarms.models.petals import Petals
 from swarms.models.mistral import Mistral
 from swarms.models.openai_models import OpenAI, AzureOpenAI, OpenAIChat
+from swarms.models.fuyu import Fuyu

swarms/models/fuyu.py

@@ -0,0 +1,65 @@
+"""Fuyu model by Kye"""
+from transformers import (
+    FuyuForCausalLM,
+    AutoTokenizer,
+    FuyuProcessor,
+    FuyuImageProcessor,
+)
+from PIL import Image
+
+
+class Fuyu:
+    """
+    Fuyu model by Adept
+
+
+    Parameters
+    ----------
+    pretrained_path : str
+        Path to the pretrained model
+    device_map : str
+        Device to use for the model
+    max_new_tokens : int
+        Maximum number of tokens to generate
+    
+    Examples
+    --------
+    >>> fuyu = Fuyu()
+    >>> fuyu("Hello, my name is", "path/to/image.png")
+
+    
+    
+    """
+    def __init__(
+        self,
+        pretrained_path: str = "adept/fuyu-8b",
+        device_map: str = "cuda",
+        max_new_tokens: int = 7,
+    ):
+        self.pretrained_path = pretrained_path
+        self.device_map = device_map
+        self.max_new_tokens = max_new_tokens
+
+        self.tokenizer = AutoTokenizer.from_pretrained(pretrained_path)
+        self.image_processor = FuyuImageProcessor()
+        self.processor = FuyuProcessor(
+            image_procesor=self.image_processor, tokenizer=self.tokenizer
+        )
+        self.model = FuyuForCausalLM.from_pretrained(
+            pretrained_path, device_map=device_map
+        )
+
+    def __call__(self, text, img_path):
+        """Call the model"""
+        image_pil = Image.open(img_path)
+        model_inputs = self.processor(
+            text=text, images=[image_pil], device=self.device_map
+        )
+
+        for k, v in model_inputs.items():
+            model_inputs[k] = v.to(self.device_map)
+
+        output = self.model.generate(
+            **model_inputs, max_new_tokens=self.fmax_new_tokens
+        )
+        text = self.processor.batch_decode(output[:, -7:], skip_special_tokens=True)

swarms/utils/__init__.py

@@ -1,3 +1,2 @@
-
 from swarms.utils.display_markdown import display_markdown_message
 from swarms.utils.futures import execute_futures_dict

tests/memory/pinecone.py

@@ -54,9 +54,9 @@ def test_query():
 
 
 def test_create_index():
-    with patch("pinecone.init"), patch(
+    with patch("pinecone.init"), patch("pinecone.Index"), patch(
-        "pinecone.Index"
+        "pinecone.create_index"
-    ), patch("pinecone.create_index") as MockCreateIndex:
+    ) as MockCreateIndex:
         store = PineconeVectorStore(
             api_key=api_key, index_name="test_index", environment="test_env"
         )