From 3443832888f71874817446b9aaa3ecbd1ed08ed7 Mon Sep 17 00:00:00 2001 From: Artem-Darius Weber Date: Mon, 28 Jul 2025 20:41:44 +0300 Subject: [PATCH] =?UTF-8?q?=D0=B8=D1=81=D0=BF=D1=80=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=B0=20=D1=80=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20gra?= =?UTF-8?q?dio=20app=20=D0=B7=D0=B0=D0=B2=D0=B8=D1=81=D0=B0=D0=BD=D0=B8?= =?UTF-8?q?=D1=8F,=20=D1=83=D1=81=D1=82=D0=B0=D0=BD=D0=BE=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=20AdminVector=20=D0=B4=D0=BB=D1=8F=20=D0=B4=D0=BE?= =?UTF-8?q?=D1=81=D1=82=D1=83=D0=BF=D0=B0=20=D0=BA=20ChromaDB,=20=D0=B8?= =?UTF-8?q?=D1=81=D0=BF=D1=80=D0=B0=D0=B2=D0=BB=D0=B5=D0=BD=D0=B0=20=D0=BE?= =?UTF-8?q?=D1=88=D0=B8=D0=B1=D0=BA=D0=B0=20=D1=80=D0=B0=D0=B7=D0=BC=D0=B5?= =?UTF-8?q?=D1=80=D0=B0=20=D0=B2=D0=B5=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=20=D0=B8=D0=B7-=D0=B7=D0=B0=20=D0=BA=D0=BE=D1=82=D0=BE=D1=80?= =?UTF-8?q?=D0=BE=D0=B9=20=D0=BB=D0=B8=D1=86=D0=BE=20=D0=BD=D0=B5=20=D1=80?= =?UTF-8?q?=D0=B5=D0=B3=D0=B5=D1=81=D1=82=D1=80=D0=B8=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B0=D0=BB=D0=BE=D1=81=D1=8C=20=D0=B2=20vector=20db?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.yml | 46 ++++++++++++++++++++++++-- main.py | 22 ++++++++++-- src/infrastructure/config.py | 2 +- src/infrastructure/ml/detectors.py | 19 ++++++++++- src/infrastructure/ml/recognizers.py | 17 +++++++++- src/infrastructure/storage/chromadb.py | 28 ++++++++++++---- 6 files changed, 121 insertions(+), 13 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3b86ca7..8494c05 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,7 @@ services: - face_recognition_net chromadb: - image: chromadb/chroma:latest + image: chromadb/chroma:0.4.15 container_name: face_recognition_chroma restart: always ports: @@ -38,7 +38,7 @@ services: container_name: face_recognition_app restart: always ports: - - "7860:7860" + - "8080:8080" volumes: - ./data:/app/data - ./models:/app/models @@ -62,9 +62,51 @@ services: count: 1 capabilities: [gpu] + postgres: + image: postgres:15 + container_name: vectoradmin_postgres + restart: unless-stopped + environment: + POSTGRES_USER: vectoradmin + POSTGRES_PASSWORD: "your-db-password" + POSTGRES_DB: vdbms + volumes: + - pgdata:/var/lib/postgresql/data + networks: + - face_recognition_net + + vector-admin: + image: mintplexlabs/vectoradmin:latest + container_name: vectoradmin_ui + restart: unless-stopped + ports: + - "3001:3001" + depends_on: + - postgres + - chromadb + environment: + # Порт, на котором слушает UI + SERVER_PORT: 3001 + # Секреты для JWT и Inngest + JWT_SECRET: "your-jwt-secret" + INNGEST_EVENT_KEY: "background_workers" + INNGEST_SIGNING_KEY: "your-inngest-signing-key" + INNGEST_LANDING_PAGE: "true" + # База данных для метаданных VectorAdmin + DATABASE_CONNECTION_STRING: "postgresql://vectoradmin:your-db-password@postgres:5432/vdbms" + # Подключение к ChromaDB (основное хранилище векторов) + VECTORS_PROVIDER_URL: "http://chromadb:8000" + VECTORS_PROVIDER_TYPE: "chroma" + # Автоматическая настройка администратора + SYS_EMAIL: "admin@vectoradmin.com" + SYS_PASSWORD: "admin123" + networks: + - face_recognition_net + volumes: mongo_data: chroma_data: + pgdata: networks: face_recognition_net: diff --git a/main.py b/main.py index 8a9f5ab..da33427 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,9 @@ from pathlib import Path +import warnings + +# Suppress numpy deprecation warnings from InsightFace +warnings.filterwarnings("ignore", message=".*rcond parameter will change.*") +warnings.filterwarnings("ignore", category=FutureWarning, module="insightface") import gradio as gr import numpy as np from typing import Optional, Tuple, List @@ -50,6 +55,19 @@ async def initialize_app(): identify_use_case=identify_use_case ) +async def main(): + app = await initialize_app() + app.launch( + server_name="0.0.0.0", + server_port=8080, + share=False, + debug=True, + # enable_queue=False, + show_error=True, + quiet=False, + prevent_thread_lock=False, + max_threads=40 + ) + if __name__ == "__main__": - app = asyncio.run(initialize_app()) - app.launch(server_name="0.0.0.0", server_port=7860) \ No newline at end of file + asyncio.run(main()) \ No newline at end of file diff --git a/src/infrastructure/config.py b/src/infrastructure/config.py index 51e7be4..c169f5b 100644 --- a/src/infrastructure/config.py +++ b/src/infrastructure/config.py @@ -11,7 +11,7 @@ class Settings(BaseSettings): QUALITY_THRESHOLD: float = 0.7 VERIFICATION_THRESHOLD: float = 0.8 - IDENTIFICATION_THRESHOLD: float = 0.75 + IDENTIFICATION_THRESHOLD: float = 0.65 MAX_FACES_PER_USER: int = 10 diff --git a/src/infrastructure/ml/detectors.py b/src/infrastructure/ml/detectors.py index ae27ebf..9c52fed 100644 --- a/src/infrastructure/ml/detectors.py +++ b/src/infrastructure/ml/detectors.py @@ -11,7 +11,24 @@ class RetinaFaceDetector: def __init__(self): detector_logger.info("Initializing RetinaFace detector") try: - self.app = FaceAnalysis(providers=['CUDAExecutionProvider', 'CPUExecutionProvider']) + # Import onnxruntime to check available providers + import onnxruntime as ort + + # Check available providers and configure accordingly + available_providers = ort.get_available_providers() + providers = [] + + if 'CUDAExecutionProvider' in available_providers: + providers.append('CUDAExecutionProvider') + detector_logger.info("CUDA provider available for face detection") + elif 'CPUExecutionProvider' in available_providers: + providers.append('CPUExecutionProvider') + detector_logger.info("Using CPU provider for face detection") + else: + providers = available_providers + detector_logger.warning(f"Using available providers: {available_providers}") + + self.app = FaceAnalysis(providers=providers) self.app.prepare(ctx_id=0, det_size=(640, 640)) detector_logger.info("RetinaFace detector initialized successfully", providers=self.app.models.keys() if hasattr(self.app, 'models') else "unknown") diff --git a/src/infrastructure/ml/recognizers.py b/src/infrastructure/ml/recognizers.py index b54db3f..8543490 100644 --- a/src/infrastructure/ml/recognizers.py +++ b/src/infrastructure/ml/recognizers.py @@ -13,9 +13,24 @@ class SFaceRecognizer: recognizer_logger.info("Initializing SFace recognizer") try: self.model_path = self._download_model() + + # Check available providers and configure accordingly + available_providers = ort.get_available_providers() + providers = [] + + if 'CUDAExecutionProvider' in available_providers: + providers.append('CUDAExecutionProvider') + recognizer_logger.info("CUDA provider available for inference") + elif 'CPUExecutionProvider' in available_providers: + providers.append('CPUExecutionProvider') + recognizer_logger.info("Using CPU provider for inference") + else: + providers = available_providers + recognizer_logger.warning(f"Using available providers: {available_providers}") + self.session = ort.InferenceSession( self.model_path, - providers=['CUDAExecutionProvider', 'CPUExecutionProvider'] + providers=providers ) self.input_name = self.session.get_inputs()[0].name self.output_name = self.session.get_outputs()[0].name diff --git a/src/infrastructure/storage/chromadb.py b/src/infrastructure/storage/chromadb.py index 746903f..5f14606 100644 --- a/src/infrastructure/storage/chromadb.py +++ b/src/infrastructure/storage/chromadb.py @@ -23,12 +23,17 @@ class ChromaDBVectorStore(VectorStore): max_retries = 30 for i in range(max_retries): try: - response = requests.get(f"http://{host}:{port}/api/v1/heartbeat") - if response.status_code == 200: + # Try v2 API first, then fallback to basic connection test + try: + response = requests.get(f"http://{host}:{port}/api/v1/version") + except: + response = requests.get(f"http://{host}:{port}") + + if response.status_code in [200, 404]: # 404 is OK, means server is responding vector_store_logger.info("ChromaDB is ready!") break except Exception as e: - vector_store_logger.debug(f"ChromaDB heartbeat check failed (attempt {i+1})", error=e) + vector_store_logger.debug(f"ChromaDB connection check failed (attempt {i+1})", error=e) pass if i == max_retries - 1: vector_store_logger.warning("ChromaDB not responding, attempting to connect anyway...") @@ -41,7 +46,10 @@ class ChromaDBVectorStore(VectorStore): try: vector_store_logger.debug("Trying direct ChromaDB connection...") self.client = chromadb.HttpClient(host=host, port=port) - self.collection = self.client.get_or_create_collection(name="face_embeddings") + self.collection = self.client.get_or_create_collection( + name="face_embeddings", + metadata={"hnsw:space": "cosine"} + ) vector_store_logger.info("Direct ChromaDB connection successful!") connection_successful = True except Exception as e: @@ -53,7 +61,10 @@ class ChromaDBVectorStore(VectorStore): vector_store_logger.debug("Trying ChromaDB connection with settings...") settings = chromadb.config.Settings(allow_reset=True) self.client = chromadb.HttpClient(host=host, port=port, settings=settings) - self.collection = self.client.get_or_create_collection(name="face_embeddings") + self.collection = self.client.get_or_create_collection( + name="face_embeddings", + metadata={"hnsw:space": "cosine"} + ) vector_store_logger.info("ChromaDB connection with settings successful!") connection_successful = True except Exception as e: @@ -65,7 +76,10 @@ class ChromaDBVectorStore(VectorStore): vector_store_logger.debug("Trying persistent client fallback...") import tempfile self.client = chromadb.PersistentClient(path=tempfile.mkdtemp()) - self.collection = self.client.get_or_create_collection(name="face_embeddings") + self.collection = self.client.get_or_create_collection( + name="face_embeddings", + metadata={"hnsw:space": "cosine"} + ) vector_store_logger.warning("Using persistent client fallback!") connection_successful = True except Exception as e: @@ -146,6 +160,8 @@ class ChromaDBVectorStore(VectorStore): face_id = results['ids'][0][i] distance = results['distances'][0][i] metadata = results['metadatas'][0][i] + # For cosine distance, similarity = 1 - distance + # Cosine distance ranges from 0 (identical) to 2 (opposite) similarity = 1 - distance similar_faces.append((face_id, similarity, metadata))