|
|
@ -115,9 +115,9 @@ class VectorStorage:
|
|
|
|
processed_files = set() # Track processed files
|
|
|
|
processed_files = set() # Track processed files
|
|
|
|
if os.path.isfile(metadata_file):
|
|
|
|
if os.path.isfile(metadata_file):
|
|
|
|
with open(
|
|
|
|
with open(
|
|
|
|
metadata_file, "r", encoding="utf-8"
|
|
|
|
metadata_file, "r",
|
|
|
|
) as metadata_file:
|
|
|
|
) as metadata_file_handle:
|
|
|
|
metadata = dict[str, str](json.load(metadata_file))
|
|
|
|
metadata = dict[str, str](json.load(metadata_file_handle))
|
|
|
|
processed_files = {
|
|
|
|
processed_files = {
|
|
|
|
entry["file"]
|
|
|
|
entry["file"]
|
|
|
|
for entry in metadata.get("processed_files", [])
|
|
|
|
for entry in metadata.get("processed_files", [])
|
|
|
@ -201,9 +201,9 @@ class VectorStorage:
|
|
|
|
|
|
|
|
|
|
|
|
# Save metadata to the metadata.json file
|
|
|
|
# Save metadata to the metadata.json file
|
|
|
|
with open(
|
|
|
|
with open(
|
|
|
|
metadata_file, "w", encoding="utf-8"
|
|
|
|
metadata_file, "w"
|
|
|
|
) as metadata_file:
|
|
|
|
) as metadata_file_handle: # Changed variable name here
|
|
|
|
json.dump(metadata, metadata_file, indent=4)
|
|
|
|
json.dump(metadata, metadata_file_handle, indent=4)
|
|
|
|
|
|
|
|
|
|
|
|
print(
|
|
|
|
print(
|
|
|
|
f"Loaded {len(documents)} documents for directory '{subdir}'."
|
|
|
|
f"Loaded {len(documents)} documents for directory '{subdir}'."
|
|
|
@ -273,4 +273,4 @@ class VectorStorage:
|
|
|
|
"swarms"
|
|
|
|
"swarms"
|
|
|
|
]
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
|
|
return retriever
|
|
|
|
return retriever
|