From 1f976cf297b10dbed621ea5d44883275d3dbe12f Mon Sep 17 00:00:00 2001 From: q Date: Sat, 18 Apr 2026 15:26:11 +0300 Subject: [PATCH] Add logviewer project and fix Docker imports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add logviewer/: Dozzle web UI (port 9999) + analyze.py CLI tool - docker-compose.yml: add json-file logging with rotation and labels for index/search - Fix Dockerfiles: COPY *.py . so all modules are included in image - Convert all relative imports to flat absolute imports for Docker flat layout - Rename index/schemas.py → index/index_schemas.py to avoid module name collision with search/schemas.py in test runner - Update all tests to add service dir to sys.path and use flat imports Co-Authored-By: Claude Sonnet 4.6 --- .codex | 0 doc/curl_api_test.md | 366 +++++++++++++++++++++++++ docker-compose.yml | 20 +- index/Dockerfile | 2 +- index/chunking.py | 6 +- index/{schemas.py => index_schemas.py} | 0 index/main.py | 6 +- index/rendering.py | 2 +- index/sparse.py | 2 +- kredit.md | 3 + logviewer/analyze.py | 179 ++++++++++++ logviewer/docker-compose.yml | 12 + search/Dockerfile | 2 +- search/aggregation.py | 4 +- search/main.py | 12 +- search/query_builder.py | 4 +- search/rerank.py | 4 +- search/retrieval.py | 4 +- tests/test_aggregation.py | 8 +- tests/test_chunking.py | 10 +- tests/test_cleaning.py | 8 +- tests/test_query_builder.py | 8 +- tests/test_rendering.py | 8 +- 23 files changed, 629 insertions(+), 41 deletions(-) create mode 100644 .codex create mode 100644 doc/curl_api_test.md rename index/{schemas.py => index_schemas.py} (100%) create mode 100644 kredit.md create mode 100755 logviewer/analyze.py create mode 100644 logviewer/docker-compose.yml diff --git a/.codex b/.codex new file mode 100644 index 0000000..e69de29 diff --git a/doc/curl_api_test.md b/doc/curl_api_test.md new file mode 100644 index 0000000..11cb5f9 --- /dev/null +++ b/doc/curl_api_test.md @@ -0,0 +1,366 @@ +# Curl API Test + +## Sources + +- Canonical contracts: `doc/ТЗ_на_хакатон_Индексация_и_поиск_по_сообщениям.pdf` +- Runnable examples and local launch notes: `README.md` +- Actual local wiring: `docker-compose.yml` + +PDF gives the strict request/response schemas for `POST /index`, `POST /sparse_embedding`, and `POST /search`. +`README.md` adds ready curl examples for the minimal requests. +This file normalizes both into checks against the current local compose stack. + +## Compose Wiring + +- `index`: `http://localhost:8001` +- `search`: `http://localhost:8002` +- `qdrant`: `http://localhost:6334` +- Inside compose, services use `QDRANT_URL=http://qdrant:6333` +- Collection name from `.env`: `evaluation` +- Vector names from `.env`: `dense` and `sparse` + +Note: current `docker-compose.yml` publishes Qdrant as `6334:6333`, while `README.md` still says `localhost:6333`. For local checks in this repo state, use `localhost:6334`. + +## Extracted API Requests + +### `GET /health` + +Both services must answer `200 OK`. + +```bash +curl -sS http://localhost:8001/health +curl -sS http://localhost:8002/health +``` + +Expected shape: + +```json +{"status":"ok"} +``` + +### `POST /index` + +Schema from the PDF: + +- body root: `data` +- `data.chat` +- `data.overlap_messages[]` +- `data.new_messages[]` + +Runnable request: + +```bash +curl -sS -X POST http://localhost:8001/index \ + -H 'Content-Type: application/json' \ + -d '{ + "data": { + "chat": { + "id": "chat-1", + "name": "Go Nova", + "sn": "chat-1@chat.agent", + "type": "channel", + "is_public": true + }, + "overlap_messages": [ + { + "id": "1", + "time": 1710000000, + "text": "Обсуждаем релиз Go", + "sender_id": "u1", + "file_snippets": "", + "parts": [], + "mentions": [], + "member_event": null, + "is_system": false, + "is_hidden": false, + "is_forward": false, + "is_quote": false + } + ], + "new_messages": [ + { + "id": "2", + "time": 1710000060, + "text": "Релиз Go перенесли на следующую неделю", + "sender_id": "u2", + "file_snippets": "", + "parts": [], + "mentions": [], + "member_event": null, + "is_system": false, + "is_hidden": false, + "is_forward": false, + "is_quote": false + } + ] + } + }' +``` + +Observed response: + +```json +{ + "results": [ + { + "page_content": "u1: Обсуждаем релиз Go\nu2: Релиз Go перенесли на следующую неделю", + "dense_content": "[2024-03-09 16:00] sender:u1\nОбсуждаем релиз Go\n[2024-03-09 16:01] sender:u2\nРелиз Go перенесли на следующую неделю", + "sparse_content": "u1 Обсуждаем релиз Go u2 Релиз Go перенесли на следующую неделю", + "message_ids": ["2"] + } + ] +} +``` + +Note: overlap messages are used as context, but are not included in returned `message_ids`. + +### `POST /sparse_embedding` + +Schema from the PDF: + +- body root: `texts: string[]` + +Runnable request: + +```bash +curl -sS -X POST http://localhost:8001/sparse_embedding \ + -H 'Content-Type: application/json' \ + -d '{ + "texts": [ + "Релиз Go перенесли на следующую неделю", + "VK GPT обсуждали в отдельном чате" + ] + }' +``` + +Observed response: + +```json +{ + "vectors": [ + { + "indices": [275068001, 108710752, 842257583, 1159207840, 2129888840, 703082301], + "values": [1.6652868125369606, 1.6652868125369606, 1.6652868125369606, 1.6652868125369606, 1.6652868125369606, 1.6652868125369606] + }, + { + "indices": [73209461, 751565418, 59863655, 1856729543, 2036701913, 1943620510], + "values": [1.6652868125369606, 1.6652868125369606, 1.6652868125369606, 1.6652868125369606, 1.6652868125369606, 1.6652868125369606] + } + ] +} +``` + +### `POST /search` + +Minimal request from `README.md`: + +```bash +curl -sS -X POST http://localhost:8002/search \ + -H 'Content-Type: application/json' \ + -d '{ + "question": { + "text": "Что писали про релиз Go?" + } + }' +``` + +Full schema from the PDF: + +```json +{ + "question": { + "text": "Что писали про релиз Go?", + "asker": "u2", + "asked_on": "2024-03-09", + "variants": ["релиз go перенесли?", "обсуждение релиза go"], + "hyde": ["В чате пишут, что релиз Go перенесли на следующую неделю."], + "keywords": ["релиз", "Go", "перенесли"], + "entities": { + "people": ["u2"], + "emails": [], + "documents": [], + "names": ["Go"], + "links": [] + }, + "date_mentions": ["следующая неделя", "2024-03-09"], + "date_range": { + "from": "2024-03-09T00:00:00Z", + "to": "2024-03-10T00:00:00Z" + }, + "search_text": "релиз Go перенесли на следующую неделю" + } +} +``` + +## Checks Run + +### 1. Health checks + +Commands: + +```bash +curl -sS http://localhost:8001/health +curl -sS http://localhost:8002/health +``` + +Observed: + +```json +{"status":"ok"} +{"status":"ok"} +``` + +### 2. Qdrant collection exists, but starts empty + +Command: + +```bash +curl -sS http://localhost:6334/collections/evaluation +``` + +Observed before manual insert: + +- `points_count: 0` +- `indexed_vectors_count: 0` + +This matches the README note that local compose creates the collection, but the template flow does not automatically upsert `/index` output into Qdrant. + +### 3. `/index` works + +Observed: + +- HTTP request completed successfully +- service returned one chunk +- returned fields match the contract: `page_content`, `dense_content`, `sparse_content`, `message_ids` + +### 4. `/sparse_embedding` works + +Observed: + +- HTTP request completed successfully +- response returned `vectors[]` +- each vector contains `indices[]` and `values[]` + +### 5. `/search` on an empty collection returns an empty result + +Command: + +```bash +curl -sS -X POST http://localhost:8002/search \ + -H 'Content-Type: application/json' \ + -d '{"question":{"text":"Что писали про релиз Go?"}}' +``` + +Observed: + +```json +{"results":[]} +``` + +This is expected while `evaluation` has no points. + +### 6. Manual Qdrant upsert for end-to-end smoke test + +To verify `/search` end-to-end, I inserted one synthetic point into local Qdrant with: + +- point id `1001` +- dummy dense vector of size `1024` +- sparse vector under field `sparse` +- payload containing `page_content` and `metadata.message_ids=["2"]` + +Command: + +```bash +vec=$(awk 'BEGIN{for(i=0;i<1024;i++) printf "%s%d", (i?",":""), (i==0)}') +curl -sS -X PUT 'http://localhost:6334/collections/evaluation/points?wait=true' \ + -H 'Content-Type: application/json' \ + -d "{\"points\":[{\"id\":1001,\"vector\":{\"dense\":[${vec}],\"sparse\":{\"indices\":[1],\"values\":[1.0]}},\"payload\":{\"page_content\":\"u1: Обсуждаем релиз Go\\nu2: Релиз Go перенесли на следующую неделю\",\"metadata\":{\"message_ids\":[\"2\"],\"participants\":[\"u1\",\"u2\"],\"start\":\"2024-03-09T16:00:00Z\",\"end\":\"2024-03-09T16:01:00Z\",\"chat_id\":\"chat-1\",\"chat_name\":\"Go Nova\",\"chat_type\":\"channel\",\"chat_sn\":\"chat-1@chat.agent\"}}}]}" +``` + +Observed: + +```json +{"result":{"operation_id":0,"status":"completed"},"status":"ok","time":0.008234969} +``` + +Collection state after insert: + +- `points_count: 1` +- `indexed_vectors_count: 1` + +### 7. `/search` works after one point is present + +Minimal request: + +```bash +curl -sS -X POST http://localhost:8002/search \ + -H 'Content-Type: application/json' \ + -d '{"question":{"text":"Что писали про релиз Go?"}}' +``` + +Observed: + +```json +{"results":[{"message_ids":["2"]}]} +``` + +Enriched request without `date_range`: + +```bash +curl -sS -X POST http://localhost:8002/search \ + -H 'Content-Type: application/json' \ + -d '{ + "question": { + "text": "Что писали про релиз Go?", + "asker": "u2", + "asked_on": "2024-03-09", + "variants": ["релиз go перенесли?", "обсуждение релиза go"], + "hyde": ["В чате пишут, что релиз Go перенесли на следующую неделю."], + "keywords": ["релиз", "Go", "перенесли"], + "entities": { + "people": ["u2"], + "emails": [], + "documents": [], + "names": ["Go"], + "links": [] + }, + "date_mentions": ["следующая неделя", "2024-03-09"], + "search_text": "релиз Go перенесли на следующую неделю" + } + }' +``` + +Observed: + +```json +{"results":[{"message_ids":["2"]}]} +``` + +### 8. Defect: `date_range` request currently fails + +The full PDF-shaped request with ISO timestamps in `question.date_range` does not work in the current implementation. + +Observed: + +```json +{ + "detail": "2 validation errors for Range\ngte\n Input should be a valid number, unable to parse string as a number [type=float_parsing, input_value='2024-03-09T00:00:00Z', input_type=str]\n For further information visit https://errors.pydantic.dev/2.12/v/float_parsing\nlte\n Input should be a valid number, unable to parse string as a number [type=float_parsing, input_value='2024-03-10T00:00:00Z', input_type=str]\n For further information visit https://errors.pydantic.dev/2.12/v/float_parsing" +} +``` + +Interpretation: + +- the public request schema accepts ISO date strings +- current `search` code tries to pass them into a numeric `qdrant_client.models.Range` +- so `date_range` is a real runtime bug in the current local build + +## Bottom Line + +- `index /health`: OK +- `search /health`: OK +- `POST /index`: OK +- `POST /sparse_embedding`: OK +- `POST /search` on empty collection: OK, returns empty list +- `POST /search` after one test point is inserted: OK +- `POST /search` with enriched request excluding `date_range`: OK +- `POST /search` with `date_range` from the PDF schema: FAILS in current implementation diff --git a/docker-compose.yml b/docker-compose.yml index 1518d0c..96c4b49 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ services: qdrant: image: qdrant/qdrant:v1.14.1 ports: - - "6333:6333" + - "6334:6333" qdrant-init: image: curlimages/curl:8.12.1 @@ -44,6 +44,15 @@ services: - qdrant ports: - "8001:8000" + logging: + driver: json-file + options: + max-size: "20m" + max-file: "5" + tag: "index-service" + labels: "service" + labels: + - "service=index-service" search: build: @@ -55,3 +64,12 @@ services: condition: service_completed_successfully ports: - "8002:8000" + logging: + driver: json-file + options: + max-size: "20m" + max-file: "5" + tag: "search-service" + labels: "service" + labels: + - "service=search-service" diff --git a/index/Dockerfile b/index/Dockerfile index 06878c2..49b2a32 100644 --- a/index/Dockerfile +++ b/index/Dockerfile @@ -5,7 +5,7 @@ WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -COPY main.py . +COPY *.py . ENV HOST=0.0.0.0 ENV PORT=8000 diff --git a/index/chunking.py b/index/chunking.py index fbf4352..6b76f60 100644 --- a/index/chunking.py +++ b/index/chunking.py @@ -1,8 +1,8 @@ """Message-based chunking with window by count, length, and time gap.""" -from .cleaning import CleanedMessage, clean_message -from .rendering import render_dense_content, render_page_content, render_sparse_content -from .schemas import IndexAPIItem, Message +from cleaning import CleanedMessage, clean_message +from rendering import render_dense_content, render_page_content, render_sparse_content +from index_schemas import IndexAPIItem, Message WINDOW_MAX_MESSAGES = 10 WINDOW_MAX_CHARS = 2048 diff --git a/index/schemas.py b/index/index_schemas.py similarity index 100% rename from index/schemas.py rename to index/index_schemas.py diff --git a/index/main.py b/index/main.py index 38bce10..2fa512d 100644 --- a/index/main.py +++ b/index/main.py @@ -7,9 +7,9 @@ from fastapi import FastAPI, Request from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse -from .chunking import build_chunks -from .schemas import IndexAPIRequest, IndexAPIResponse, SparseEmbeddingRequest -from .sparse import embed_sparse_texts +from chunking import build_chunks +from index_schemas import IndexAPIRequest, IndexAPIResponse, SparseEmbeddingRequest +from sparse import embed_sparse_texts HOST = os.getenv("HOST", "0.0.0.0") PORT = int(os.getenv("PORT", "8004")) diff --git a/index/rendering.py b/index/rendering.py index 2d46ee0..8237ef3 100644 --- a/index/rendering.py +++ b/index/rendering.py @@ -2,7 +2,7 @@ import datetime -from .cleaning import CleanedMessage +from cleaning import CleanedMessage def _format_time(ts: int) -> str: diff --git a/index/sparse.py b/index/sparse.py index 3c6313d..26404c7 100644 --- a/index/sparse.py +++ b/index/sparse.py @@ -2,7 +2,7 @@ import logging import os from functools import lru_cache -from .schemas import SparseVector +from index_schemas import SparseVector SPARSE_MODEL_NAME = "Qdrant/bm25" FASTEMBED_CACHE_PATH = "/models/fastembed" diff --git a/kredit.md b/kredit.md new file mode 100644 index 0000000..12f87f3 --- /dev/null +++ b/kredit.md @@ -0,0 +1,3 @@ +team_id: 35230 +vk login: 56aa86799bb9edc4 +vk password: edd89cea9ed0734d00ba6904cf7475d7 diff --git a/logviewer/analyze.py b/logviewer/analyze.py new file mode 100755 index 0000000..291442e --- /dev/null +++ b/logviewer/analyze.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +""" +CLI tool to query and analyze Docker logs from index-service and search-service. + +Usage: + python3 analyze.py # tail both services, live + python3 analyze.py --service search # filter by service + python3 analyze.py --level ERROR # filter by log level + python3 analyze.py --grep "qdrant" # grep in log message + python3 analyze.py --last 100 # last N lines per service + python3 analyze.py --since 10m # since N minutes/hours ago (e.g. 10m, 2h) + python3 analyze.py --stats # show request count / error rate summary +""" + +import argparse +import json +import re +import subprocess +import sys +from datetime import datetime, timezone + +SERVICES = { + "index": "hackaton-index-1", + "search": "hackaton-search-1", +} + + +def _run(cmd: list[str], **kwargs) -> subprocess.CompletedProcess: + return subprocess.run(cmd, capture_output=True, text=True, **kwargs) + + +def _detect_container_name(service: str) -> str: + """Find the actual running container name for a service.""" + candidates = [ + f"hackaton-{service}-1", + f"hackaton_{service}_1", + f"{service}-1", + f"{service}_1", + ] + result = _run(["docker", "ps", "--format", "{{.Names}}"]) + running = result.stdout.splitlines() + for name in candidates: + if name in running: + return name + # fallback: try matching by label + result2 = _run( + ["docker", "ps", "--filter", f"label=service={service}-service", "--format", "{{.Names}}"] + ) + names = result2.stdout.strip().splitlines() + if names: + return names[0] + return candidates[0] + + +def _docker_logs(container: str, since: str | None, last: int) -> list[str]: + cmd = ["docker", "logs", "--timestamps"] + if since: + cmd += ["--since", since] + if last: + cmd += ["--tail", str(last)] + cmd.append(container) + result = _run(cmd) + lines = (result.stdout + result.stderr).splitlines() + return lines + + +def _parse_line(raw: str) -> dict: + """Try to extract structured fields from a log line.""" + # Docker prepends an RFC3339 timestamp + ts_match = re.match(r"^(\d{4}-\d{2}-\d{2}T[\d:.+Z-]+)\s+(.*)", raw) + ts = "" + message = raw + if ts_match: + ts = ts_match.group(1) + message = ts_match.group(2) + + level = "INFO" + for lvl in ("CRITICAL", "ERROR", "WARNING", "WARN", "INFO", "DEBUG"): + if lvl in message.upper(): + level = lvl if lvl != "WARN" else "WARNING" + break + + return {"ts": ts, "level": level, "message": message, "raw": raw} + + +def _color(level: str) -> str: + return { + "ERROR": "\033[31m", + "CRITICAL": "\033[35m", + "WARNING": "\033[33m", + "INFO": "\033[0m", + "DEBUG": "\033[36m", + }.get(level, "\033[0m") + + +RESET = "\033[0m" + + +def cmd_tail(args: argparse.Namespace) -> None: + services = [args.service] if args.service else list(SERVICES.keys()) + for svc in services: + container = _detect_container_name(svc) + lines = _docker_logs(container, args.since, args.last) + print(f"\n{'='*60}") + print(f" {svc.upper()} SERVICE ({container})") + print(f"{'='*60}") + count = 0 + for raw in lines: + parsed = _parse_line(raw) + if args.level and parsed["level"] != args.level.upper(): + continue + if args.grep and args.grep.lower() not in parsed["message"].lower(): + continue + color = _color(parsed["level"]) + print(f"{color}{parsed['raw']}{RESET}") + count += 1 + print(f" → {count} lines shown") + + +def cmd_stats(args: argparse.Namespace) -> None: + services = [args.service] if args.service else list(SERVICES.keys()) + for svc in services: + container = _detect_container_name(svc) + lines = _docker_logs(container, args.since, args.last) + + counts: dict[str, int] = {"ERROR": 0, "WARNING": 0, "INFO": 0, "DEBUG": 0, "CRITICAL": 0} + requests = 0 + errors_5xx = 0 + + for raw in lines: + parsed = _parse_line(raw) + lvl = parsed["level"] + counts[lvl] = counts.get(lvl, 0) + 1 + msg = parsed["message"] + if re.search(r'"(GET|POST|PUT|DELETE|PATCH)\s', msg): + requests += 1 + if re.search(r'" 5\d\d ', msg): + errors_5xx += 1 + + print(f"\n{'='*50}") + print(f" STATS: {svc.upper()} SERVICE") + print(f"{'='*50}") + print(f" Total lines : {sum(counts.values())}") + print(f" HTTP requests: {requests}") + print(f" 5xx errors : {errors_5xx}") + print(f" ERROR lines : {counts['ERROR']}") + print(f" WARNING lines: {counts['WARNING']}") + print(f" INFO lines : {counts['INFO']}") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Analyze Docker logs for index-service and search-service", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument("--service", choices=list(SERVICES.keys()), help="Filter by service") + parser.add_argument("--level", help="Filter by log level (ERROR, WARNING, INFO, DEBUG)") + parser.add_argument("--grep", help="Substring filter on log message") + parser.add_argument("--last", type=int, default=200, help="Last N lines per service (default 200)") + parser.add_argument("--since", help="Show logs since duration (e.g. 10m, 2h, 30s)") + parser.add_argument("--stats", action="store_true", help="Show stats summary instead of log lines") + + args = parser.parse_args() + + try: + if args.stats: + cmd_stats(args) + else: + cmd_tail(args) + except KeyboardInterrupt: + print("\nInterrupted.") + except FileNotFoundError: + print("ERROR: docker not found in PATH", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/logviewer/docker-compose.yml b/logviewer/docker-compose.yml new file mode 100644 index 0000000..834e47b --- /dev/null +++ b/logviewer/docker-compose.yml @@ -0,0 +1,12 @@ +services: + dozzle: + image: amir20/dozzle:latest + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + ports: + - "9999:8080" + environment: + DOZZLE_FILTER: "label=service" + DOZZLE_LEVEL: info + DOZZLE_ENABLE_ACTIONS: "false" + restart: unless-stopped diff --git a/search/Dockerfile b/search/Dockerfile index 1954c62..e3b5814 100644 --- a/search/Dockerfile +++ b/search/Dockerfile @@ -5,7 +5,7 @@ WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -COPY main.py . +COPY *.py . ENV HOST=0.0.0.0 ENV PORT=8000 diff --git a/search/aggregation.py b/search/aggregation.py index 1c7dcc8..f86549f 100644 --- a/search/aggregation.py +++ b/search/aggregation.py @@ -1,7 +1,7 @@ from typing import Any -from .config import TOP_K -from .retrieval import extract_message_ids +from config import TOP_K +from retrieval import extract_message_ids def aggregate_message_ids( diff --git a/search/main.py b/search/main.py index 7fa511c..27fbd92 100644 --- a/search/main.py +++ b/search/main.py @@ -9,8 +9,8 @@ from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse from qdrant_client import AsyncQdrantClient -from .aggregation import aggregate_message_ids -from .config import ( +from aggregation import aggregate_message_ids +from config import ( API_KEY, HOST, HTTP_MAX_RETRIES, @@ -20,7 +20,7 @@ from .config import ( logger, validate_required_env, ) -from .query_builder import ( +from query_builder import ( build_extra_dense_queries, build_primary_query, build_sparse_query, @@ -28,9 +28,9 @@ from .query_builder import ( embed_dense_multi, embed_sparse, ) -from .rerank import rerank_points -from .retrieval import qdrant_search -from .schemas import SearchAPIItem, SearchAPIRequest, SearchAPIResponse, SparseVector +from rerank import rerank_points +from retrieval import qdrant_search +from schemas import SearchAPIItem, SearchAPIRequest, SearchAPIResponse, SparseVector async def _embed_dense_with_retry(client: httpx.AsyncClient, text: str) -> list[float]: diff --git a/search/query_builder.py b/search/query_builder.py index c0c2006..1a9fa61 100644 --- a/search/query_builder.py +++ b/search/query_builder.py @@ -6,14 +6,14 @@ from functools import lru_cache import httpx from fastembed import SparseTextEmbedding -from .config import ( +from config import ( EMBEDDINGS_DENSE_MODEL, EMBEDDINGS_DENSE_URL, SPARSE_MODEL_NAME, get_upstream_kwargs, logger, ) -from .schemas import DenseEmbeddingResponse, Question, SparseVector +from schemas import DenseEmbeddingResponse, Question, SparseVector @lru_cache(maxsize=1) diff --git a/search/rerank.py b/search/rerank.py index 0bc7fae..a7c38b8 100644 --- a/search/rerank.py +++ b/search/rerank.py @@ -2,8 +2,8 @@ from typing import Any import httpx -from .config import RERANK_LIMIT, RERANKER_MODEL, RERANKER_URL, get_upstream_kwargs, logger -from .retrieval import extract_page_content +from config import RERANK_LIMIT, RERANKER_MODEL, RERANKER_URL, get_upstream_kwargs, logger +from retrieval import extract_page_content async def get_rerank_scores( diff --git a/search/retrieval.py b/search/retrieval.py index ccc62ca..bdebd06 100644 --- a/search/retrieval.py +++ b/search/retrieval.py @@ -2,7 +2,7 @@ from typing import Any from qdrant_client import AsyncQdrantClient, models -from .config import ( +from config import ( DENSE_PREFETCH_K, QDRANT_COLLECTION_NAME, QDRANT_DENSE_VECTOR_NAME, @@ -11,7 +11,7 @@ from .config import ( SPARSE_PREFETCH_K, logger, ) -from .schemas import Question, SparseVector +from schemas import Question, SparseVector def _build_filter(question: Question) -> models.Filter | None: diff --git a/tests/test_aggregation.py b/tests/test_aggregation.py index 6306e5b..6db865e 100644 --- a/tests/test_aggregation.py +++ b/tests/test_aggregation.py @@ -1,15 +1,17 @@ """Unit tests for search/aggregation.py""" import sys import os -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +_SEARCH_DIR = os.path.join(os.path.dirname(__file__), "..", "search") +sys.path.insert(0, _SEARCH_DIR) os.environ.setdefault("EMBEDDINGS_DENSE_URL", "http://localhost/embed") os.environ.setdefault("RERANKER_URL", "http://localhost/rerank") os.environ.setdefault("QDRANT_URL", "http://localhost:6333") os.environ.setdefault("API_KEY", "test-key") -from search.aggregation import aggregate_message_ids -from search.config import TOP_K +from aggregation import aggregate_message_ids +from config import TOP_K def _point(message_ids: list[str]): diff --git a/tests/test_chunking.py b/tests/test_chunking.py index 720f086..4ac4324 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -1,11 +1,13 @@ """Unit tests for index/chunking.py""" import sys import os -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) -from index.chunking import build_chunks, _split_windows, WINDOW_MAX_MESSAGES, TIME_GAP_SECONDS -from index.cleaning import CleanedMessage -from index.schemas import Message +_INDEX_DIR = os.path.join(os.path.dirname(__file__), "..", "index") +sys.path.insert(0, _INDEX_DIR) + +from chunking import build_chunks, _split_windows, WINDOW_MAX_MESSAGES, TIME_GAP_SECONDS +from cleaning import CleanedMessage +from index_schemas import Message def _make_message(id: str, time: int, text: str = "hello", **kwargs) -> Message: diff --git a/tests/test_cleaning.py b/tests/test_cleaning.py index d3688be..163e893 100644 --- a/tests/test_cleaning.py +++ b/tests/test_cleaning.py @@ -1,17 +1,19 @@ """Unit tests for index/cleaning.py""" import sys import os -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +_INDEX_DIR = os.path.join(os.path.dirname(__file__), "..", "index") +sys.path.insert(0, _INDEX_DIR) import pytest -from index.cleaning import ( +from cleaning import ( normalize_unicode, parse_file_snippets, normalize_member_event, normalize_part, clean_message, ) -from index.schemas import Message +from index_schemas import Message def _make_message(**kwargs) -> Message: diff --git a/tests/test_query_builder.py b/tests/test_query_builder.py index d1d0744..65823ea 100644 --- a/tests/test_query_builder.py +++ b/tests/test_query_builder.py @@ -1,7 +1,9 @@ """Unit tests for search/query_builder.py (pure logic only, no HTTP)""" import sys import os -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +_SEARCH_DIR = os.path.join(os.path.dirname(__file__), "..", "search") +sys.path.insert(0, _SEARCH_DIR) # Stub env vars before importing search modules os.environ.setdefault("EMBEDDINGS_DENSE_URL", "http://localhost/embed") @@ -9,8 +11,8 @@ os.environ.setdefault("RERANKER_URL", "http://localhost/rerank") os.environ.setdefault("QDRANT_URL", "http://localhost:6333") os.environ.setdefault("API_KEY", "test-key") -from search.schemas import Entities, Question -from search.query_builder import ( +from schemas import Entities, Question +from query_builder import ( build_primary_query, build_extra_dense_queries, build_sparse_query, diff --git a/tests/test_rendering.py b/tests/test_rendering.py index 545a9b6..1532f72 100644 --- a/tests/test_rendering.py +++ b/tests/test_rendering.py @@ -1,10 +1,12 @@ """Unit tests for index/rendering.py""" import sys import os -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) -from index.cleaning import CleanedMessage -from index.rendering import render_page_content, render_dense_content, render_sparse_content +_INDEX_DIR = os.path.join(os.path.dirname(__file__), "..", "index") +sys.path.insert(0, _INDEX_DIR) + +from cleaning import CleanedMessage +from rendering import render_page_content, render_dense_content, render_sparse_content def _make_cleaned(**kwargs) -> CleanedMessage: