@ -57,18 +57,22 @@ max_length = {
" ada " : 2049
" ada " : 2049
} }
def count_tokens ( model_name , text ) : def count_tokens ( model_name , text ) :
return len ( encodings [ model_name ] . encode ( text ) )
return len ( encodings [ model_name ] . encode ( text ) )
def get_max_context_length ( model_name ) : def get_max_context_length ( model_name ) :
return max_length [ model_name ]
return max_length [ model_name ]
def get_token_ids_for_task_parsing ( model_name ) : def get_token_ids_for_task_parsing ( model_name ) :
text = ''' { " task " : " text-classification " , " token-classification " , " text2text-generation " , " summarization " , " translation " , " question-answering " , " conversational " , " text-generation " , " sentence-similarity " , " tabular-classification " , " object-detection " , " image-classification " , " image-to-image " , " image-to-text " , " text-to-image " , " visual-question-answering " , " document-question-answering " , " image-segmentation " , " text-to-speech " , " text-to-video " , " automatic-speech-recognition " , " audio-to-audio " , " audio-classification " , " canny-control " , " hed-control " , " mlsd-control " , " normal-control " , " openpose-control " , " canny-text-to-image " , " depth-text-to-image " , " hed-text-to-image " , " mlsd-text-to-image " , " normal-text-to-image " , " openpose-text-to-image " , " seg-text-to-image " , " args " , " text " , " path " , " dep " , " id " , " <GENERATED>- " } '''
text = ''' { " task " : " text-classification " , " token-classification " , " text2text-generation " , " summarization " , " translation " , " question-answering " , " conversational " , " text-generation " , " sentence-similarity " , " tabular-classification " , " object-detection " , " image-classification " , " image-to-image " , " image-to-text " , " text-to-image " , " visual-question-answering " , " document-question-answering " , " image-segmentation " , " text-to-speech " , " text-to-video " , " automatic-speech-recognition " , " audio-to-audio " , " audio-classification " , " canny-control " , " hed-control " , " mlsd-control " , " normal-control " , " openpose-control " , " canny-text-to-image " , " depth-text-to-image " , " hed-text-to-image " , " mlsd-text-to-image " , " normal-text-to-image " , " openpose-text-to-image " , " seg-text-to-image " , " args " , " text " , " path " , " dep " , " id " , " <GENERATED>- " } '''
res = encodings [ model_name ] . encode ( text )
res = encodings [ model_name ] . encode ( text )
res = list ( set ( res ) )
res = list ( set ( res ) )
return res
return res
def get_token_ids_for_choose_model ( model_name ) : def get_token_ids_for_choose_model ( model_name ) :
text = ''' { " id " : " reason " } '''
text = ''' { " id " : " reason " } '''
res = encodings [ model_name ] . encode ( text )
res = encodings [ model_name ] . encode ( text )
@ -76,13 +80,7 @@ def get_token_ids_for_choose_model(model_name):
return res
return res
######### #########
parser = argparse . ArgumentParser ( ) parser = argparse . ArgumentParser ( )
parser . add_argument ( " --config " , type = str , default = " swarms/agents/workers/multi_modal_workers/omni_agent/config.yml " ) parser . add_argument ( " --config " , type = str , default = " swarms/agents/workers/multi_modal_workers/omni_agent/config.yml " )
parser . add_argument ( " --mode " , type = str , default = " cli " ) parser . add_argument ( " --mode " , type = str , default = " cli " )
@ -183,7 +181,7 @@ if inference_mode!="huggingface":
r = requests . get ( Model_Server + " /running " )
r = requests . get ( Model_Server + " /running " )
if r . status_code != 200 :
if r . status_code != 200 :
raise ValueError ( message )
raise ValueError ( message )
except :
except BaseException :
raise ValueError ( message )
raise ValueError ( message )
@ -222,6 +220,7 @@ elif "HUGGINGFACE_ACCESS_TOKEN" in os.environ and os.getenv("HUGGINGFACE_ACCESS_
else : else :
raise ValueError ( f " Incorrect HuggingFace token. Please check your { args . config } file. " )
raise ValueError ( f " Incorrect HuggingFace token. Please check your { args . config } file. " )
def convert_chat_to_completion ( data ) : def convert_chat_to_completion ( data ) :
messages = data . pop ( ' messages ' , [ ] )
messages = data . pop ( ' messages ' , [ ] )
tprompt = " "
tprompt = " "
@ -243,6 +242,7 @@ def convert_chat_to_completion(data):
data [ ' max_tokens ' ] = data . get ( ' max_tokens ' , max ( get_max_context_length ( LLM ) - count_tokens ( LLM_encoding , final_prompt ) , 1 ) )
data [ ' max_tokens ' ] = data . get ( ' max_tokens ' , max ( get_max_context_length ( LLM ) - count_tokens ( LLM_encoding , final_prompt ) , 1 ) )
return data
return data
def send_request ( data ) : def send_request ( data ) :
api_key = data . pop ( " api_key " )
api_key = data . pop ( " api_key " )
api_type = data . pop ( " api_type " )
api_type = data . pop ( " api_type " )
@ -269,6 +269,7 @@ def send_request(data):
else :
else :
return response . json ( ) [ " choices " ] [ 0 ] [ " message " ] [ " content " ] . strip ( )
return response . json ( ) [ " choices " ] [ 0 ] [ " message " ] [ " content " ] . strip ( )
def replace_slot ( text , entries ) : def replace_slot ( text , entries ) :
for key , value in entries . items ( ) :
for key , value in entries . items ( ) :
if not isinstance ( value , str ) :
if not isinstance ( value , str ) :
@ -276,6 +277,7 @@ def replace_slot(text, entries):
text = text . replace ( " {{ " + key + " }} " , value . replace ( ' " ' , " ' " ) . replace ( ' \n ' , " " ) )
text = text . replace ( " {{ " + key + " }} " , value . replace ( ' " ' , " ' " ) . replace ( ' \n ' , " " ) )
return text
return text
def find_json ( s ) : def find_json ( s ) :
s = s . replace ( " \' " , " \" " )
s = s . replace ( " \' " , " \" " )
start = s . find ( " { " )
start = s . find ( " { " )
@ -284,21 +286,24 @@ def find_json(s):
res = res . replace ( " \n " , " " )
res = res . replace ( " \n " , " " )
return res
return res
def field_extract ( s , field ) : def field_extract ( s , field ) :
try :
try :
field_rep = re . compile ( f ' { field } .*?:.*? " (.*?) " ' , re . IGNORECASE )
field_rep = re . compile ( f ' { field } .*?:.*? " (.*?) " ' , re . IGNORECASE )
extracted = field_rep . search ( s ) . group ( 1 ) . replace ( " \" " , " \' " )
extracted = field_rep . search ( s ) . group ( 1 ) . replace ( " \" " , " \' " )
except :
except BaseException :
field_rep = re . compile ( f ' { field } : \ * " (.*?) " ' , re . IGNORECASE )
field_rep = re . compile ( f ' { field } : \ * " (.*?) " ' , re . IGNORECASE )
extracted = field_rep . search ( s ) . group ( 1 ) . replace ( " \" " , " \' " )
extracted = field_rep . search ( s ) . group ( 1 ) . replace ( " \" " , " \' " )
return extracted
return extracted
def get_id_reason ( choose_str ) : def get_id_reason ( choose_str ) :
reason = field_extract ( choose_str , " reason " )
reason = field_extract ( choose_str , " reason " )
id = field_extract ( choose_str , " id " )
id = field_extract ( choose_str , " id " )
choose = { " id " : id , " reason " : reason }
choose = { " id " : id , " reason " : reason }
return id . strip ( ) , reason . strip ( ) , choose
return id . strip ( ) , reason . strip ( ) , choose
def record_case ( success , * * args ) : def record_case ( success , * * args ) :
if success :
if success :
f = open ( " logs/log_success.jsonl " , " a " )
f = open ( " logs/log_success.jsonl " , " a " )
@ -308,6 +313,7 @@ def record_case(success, **args):
f . write ( json . dumps ( log ) + " \n " )
f . write ( json . dumps ( log ) + " \n " )
f . close ( )
f . close ( )
def image_to_bytes ( img_url ) : def image_to_bytes ( img_url ) :
img_byte = io . BytesIO ( )
img_byte = io . BytesIO ( )
img_url . split ( " . " ) [ - 1 ]
img_url . split ( " . " ) [ - 1 ]
@ -315,6 +321,7 @@ def image_to_bytes(img_url):
img_data = img_byte . getvalue ( )
img_data = img_byte . getvalue ( )
return img_data
return img_data
def resource_has_dep ( command ) : def resource_has_dep ( command ) :
args = command [ " args " ]
args = command [ " args " ]
for _ , v in args . items ( ) :
for _ , v in args . items ( ) :
@ -322,6 +329,7 @@ def resource_has_dep(command):
return True
return True
return False
return False
def fix_dep ( tasks ) : def fix_dep ( tasks ) :
for task in tasks :
for task in tasks :
args = task [ " args " ]
args = task [ " args " ]
@ -335,6 +343,7 @@ def fix_dep(tasks):
task [ " dep " ] = [ - 1 ]
task [ " dep " ] = [ - 1 ]
return tasks
return tasks
def unfold ( tasks ) : def unfold ( tasks ) :
flag_unfold_task = False
flag_unfold_task = False
try :
try :
@ -361,6 +370,7 @@ def unfold(tasks):
return tasks
return tasks
def chitchat ( messages , api_key , api_type , api_endpoint ) : def chitchat ( messages , api_key , api_type , api_endpoint ) :
data = {
data = {
" model " : LLM ,
" model " : LLM ,
@ -371,6 +381,7 @@ def chitchat(messages, api_key, api_type, api_endpoint):
}
}
return send_request ( data )
return send_request ( data )
def parse_task ( context , input , api_key , api_type , api_endpoint ) : def parse_task ( context , input , api_key , api_type , api_endpoint ) :
demos_or_presteps = parse_task_demos_or_presteps
demos_or_presteps = parse_task_demos_or_presteps
messages = json . loads ( demos_or_presteps )
messages = json . loads ( demos_or_presteps )
@ -404,6 +415,7 @@ def parse_task(context, input, api_key, api_type, api_endpoint):
}
}
return send_request ( data )
return send_request ( data )
def choose_model ( input , task , metas , api_key , api_type , api_endpoint ) : def choose_model ( input , task , metas , api_key , api_type , api_endpoint ) :
prompt = replace_slot ( choose_model_prompt , {
prompt = replace_slot ( choose_model_prompt , {
" input " : input ,
" input " : input ,
@ -454,6 +466,7 @@ def response_results(input, results, api_key, api_type, api_endpoint):
}
}
return send_request ( data )
return send_request ( data )
def huggingface_model_inference ( model_id , data , task ) : def huggingface_model_inference ( model_id , data , task ) :
task_url = f " https://api-inference.huggingface.co/models/ { model_id } " # InferenceApi does not yet support some tasks
task_url = f " https://api-inference.huggingface.co/models/ { model_id } " # InferenceApi does not yet support some tasks
inference = InferenceApi ( repo_id = model_id , token = config [ " huggingface " ] [ " token " ] )
inference = InferenceApi ( repo_id = model_id , token = config [ " huggingface " ] [ " token " ] )
@ -586,6 +599,7 @@ def huggingface_model_inference(model_id, data, task):
result = { " generated audio " : f " /audios/ { name } . { type } " }
result = { " generated audio " : f " /audios/ { name } . { type } " }
return result
return result
def local_model_inference ( model_id , data , task ) : def local_model_inference ( model_id , data , task ) :
task_url = f " { Model_Server } /models/ { model_id } "
task_url = f " { Model_Server } /models/ { model_id } "
@ -732,6 +746,7 @@ def get_model_status(model_id, url, headers, queue = None):
queue . put ( ( model_id , False , None ) )
queue . put ( ( model_id , False , None ) )
return False
return False
def get_avaliable_models ( candidates , topk = 5 ) : def get_avaliable_models ( candidates , topk = 5 ) :
all_available_models = { " local " : [ ] , " huggingface " : [ ] }
all_available_models = { " local " : [ ] , " huggingface " : [ ] }
threads = [ ]
threads = [ ]
@ -766,6 +781,7 @@ def get_avaliable_models(candidates, topk=5):
return all_available_models
return all_available_models
def collect_result ( command , choose , inference_result ) : def collect_result ( command , choose , inference_result ) :
result = { " task " : command }
result = { " task " : command }
result [ " inference result " ] = inference_result
result [ " inference result " ] = inference_result
@ -945,6 +961,7 @@ def run_task(input, command, results, api_key, api_type, api_endpoint):
results [ id ] = collect_result ( command , choose , inference_result )
results [ id ] = collect_result ( command , choose , inference_result )
return True
return True
def chat_huggingface ( messages , api_key , api_type , api_endpoint , return_planning = False , return_results = False ) : def chat_huggingface ( messages , api_key , api_type , api_endpoint , return_planning = False , return_results = False ) :
start = time . time ( )
start = time . time ( )
context = messages [ : - 1 ]
context = messages [ : - 1 ]
@ -1032,6 +1049,7 @@ def chat_huggingface(messages, api_key, api_type, api_endpoint, return_planning
logger . info ( f " response: { response } " )
logger . info ( f " response: { response } " )
return answer
return answer
def test ( ) : def test ( ) :
# single round examples
# single round examples
inputs = [
inputs = [
@ -1055,6 +1073,7 @@ def test():
]
]
chat_huggingface ( messages , API_KEY , API_TYPE , API_ENDPOINT , return_planning = False , return_results = False )
chat_huggingface ( messages , API_KEY , API_TYPE , API_ENDPOINT , return_planning = False , return_results = False )
def cli ( ) : def cli ( ) :
messages = [ ]
messages = [ ]
print ( " Welcome to Jarvis! A collaborative system that consists of an LLM as the controller and numerous expert models as collaborative executors. Jarvis can plan tasks, schedule Hugging Face models, generate friendly responses based on your requests, and help you with many things. Please enter your request (`exit` to exit). " )
print ( " Welcome to Jarvis! A collaborative system that consists of an LLM as the controller and numerous expert models as collaborative executors. Jarvis can plan tasks, schedule Hugging Face models, generate friendly responses based on your requests, and help you with many things. Please enter your request (`exit` to exit). " )
Kye
1 year ago