1f976cf297
- Add logviewer/: Dozzle web UI (port 9999) + analyze.py CLI tool - docker-compose.yml: add json-file logging with rotation and labels for index/search - Fix Dockerfiles: COPY *.py . so all modules are included in image - Convert all relative imports to flat absolute imports for Docker flat layout - Rename index/schemas.py → index/index_schemas.py to avoid module name collision with search/schemas.py in test runner - Update all tests to add service dir to sys.path and use flat imports Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
31 lines
864 B
Python
31 lines
864 B
Python
import logging
|
|
import os
|
|
from functools import lru_cache
|
|
|
|
from index_schemas import SparseVector
|
|
|
|
SPARSE_MODEL_NAME = "Qdrant/bm25"
|
|
FASTEMBED_CACHE_PATH = "/models/fastembed"
|
|
|
|
logger = logging.getLogger("index-service")
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def get_sparse_model():
|
|
from fastembed import SparseTextEmbedding
|
|
|
|
logger.info("Loading sparse model %s from cache %s", SPARSE_MODEL_NAME, FASTEMBED_CACHE_PATH)
|
|
return SparseTextEmbedding(model_name=SPARSE_MODEL_NAME)
|
|
|
|
|
|
def embed_sparse_texts(texts: list[str]) -> list[SparseVector]:
|
|
model = get_sparse_model()
|
|
result: list[SparseVector] = []
|
|
for item in model.embed(texts):
|
|
result.append(
|
|
SparseVector(
|
|
indices=[int(i) for i in item.indices.tolist()],
|
|
values=[float(v) for v in item.values.tolist()],
|
|
)
|
|
)
|
|
return result
|