commit deccd982e4cc3a480de0691011cef88cf8847ffd Author: q Date: Sun Mar 8 01:40:26 2026 +0300 first commit diff --git a/README.md b/README.md new file mode 100644 index 0000000..f3415c1 --- /dev/null +++ b/README.md @@ -0,0 +1,125 @@ +# Jetson Nano Image2Text API + +Python 3.6-compatible Flask service that accepts an image and returns detected text plus optional EdgeTPU object context. + +## Features +- Accepts JPEG/PNG/WebP/BMP/TIFF/GIF (first frame) via multipart or base64 JSON. +- Optional Google Coral EdgeTPU detection for scene context (loaded once at start). +- EAST text detector (OpenCV DNN) when model exists; otherwise whole-image fallback. +- OCR via `pytesseract` with per-block confidences; configurable languages. +- Auto-selects `rus` or `eng` (via Tesseract OSD script detection) before full OCR when `OCR_LANG=eng+rus`. +- If OCR result is empty, retries with fallback language/full-frame pass to improve recall. +- Hard payload cap (10MB default) and graceful degradation if Coral/EAST unavailable. + +## System prerequisites (Ubuntu 18.04 aarch64) +```bash +# EdgeTPU runtime (choose std or max) +sudo apt-get update +sudo apt-get install -y libedgetpu1-std python3-pycoral + +# Tesseract OCR + languages +sudo apt-get install -y tesseract-ocr tesseract-ocr-eng tesseract-ocr-rus + +# OpenCV (binary from apt is preferred on Jetson) +sudo apt-get install -y python3-opencv + +# If you prefer pip OpenCV (may be large on Jetson): +# pip3 install opencv-python==4.5.3.56 +``` + +## Python dependencies +```bash +pip3 install -r requirements.txt +``` + +## Models +Place your models in `./models` (paths can be overridden via env): +- EdgeTPU SSD: `ssd_mobilenet_v2_coco_quant_postprocess_edgetpu.tflite` +- Labels: `coco_labels.txt` +- EAST text detector (optional): `frozen_east_text_detection.pb` + +### Better OCR models (tessdata_best) +```bash +sudo apt-get install -y wget +sudo mkdir -p /usr/share/tesseract-ocr/4.00/tessdata_best +sudo wget -O /usr/share/tesseract-ocr/4.00/tessdata_best/eng.traineddata https://github.com/tesseract-ocr/tessdata_best/raw/main/eng.traineddata +sudo wget -O /usr/share/tesseract-ocr/4.00/tessdata_best/rus.traineddata https://github.com/tesseract-ocr/tessdata_best/raw/main/rus.traineddata +# then run the service with: +export TESSDATA_PREFIX=/usr/share/tesseract-ocr/4.00/tessdata_best +# if you skip this, the app will auto-try common system paths +``` + +### OpenCV with CUDA (for faster EAST) +OpenCV from JetPack repos (4.x) usually ships without CUDA dnn. To use CUDA, build OpenCV 4.5+ with `-D WITH_CUDA=ON -D OPENCV_DNN_CUDA=ON`. After installing, export `USE_CUDA_DNN=true` so the service enables CUDA backend/target when available. + +## Environment variables +- `MODEL_EDGETPU_PATH` (default `./models/ssd_mobilenet_v2_coco_quant_postprocess_edgetpu.tflite`) +- `MODEL_LABELS_PATH` (default `./models/coco_labels.txt`) +- `MODEL_EAST_PATH` (default `./models/frozen_east_text_detection.pb`) +- `OCR_LANG` (default `eng+rus`; request language set for OCR) +- `OCR_AUTO_LANG` (default `true`; auto-pick `eng` or `rus` by script when `OCR_LANG` contains both) +- `OCR_CONFIG` (default `--oem 1 --psm 6`, forwarded to Tesseract) +- `TESSDATA_PREFIX` (path to tessdata / tessdata_best if you install better models) +- `OCR_UPSCALE` (default `1.5`, upscale factor before OCR; max 3.0) +- `OCR_CLAHE` (default `true`, contrast-limited adaptive histogram) +- `OCR_BILATERAL` (default `false`, light denoise) +- `OCR_BLUR` (default `0`, Gaussian kernel size; 0 disables) +- `OCR_INVERT` (default `false`, invert after threshold) +- `OCR_USE_THRESHOLD` (default `true`, adaptive threshold) +- `MAX_IMAGE_MB` (default `10`) +- `CONF_THRESHOLD` (default `0.4`) +- `RETURN_BOXES` (default `true`) +- `USE_CUDA_DNN` (default `false`, set `true` to attempt CUDA backend for EAST if OpenCV built with CUDA) +- `DEBUG_VISUAL` (default `false`; if true, response includes `debug_image_base64`. Per-request: add `?debug=1` or form field `debug=1`.) +- `SHOW_GUI` (default `false`; if true and OpenCV GUI available, service opens a window with overlays. Requires X/Wayland/VDI that supports GUI.) + +## Run +```bash +python3 app.py --host 0.0.0.0 --port 8000 +``` +Waitress or gunicorn can also serve the app if desired (not required). + +## API +- `GET /health` -> `{ "status": "ok", "coral": true/false }` +- `POST /v1/image2text` + - Multipart: field `image` + - JSON: `{ "image_base64": "" }` + +Example curl (multipart): +```bash +curl -X POST http://localhost:8000/v1/image2text \ + -F "image=@sample.jpg" \ + -F "debug=1" \ + | python -m json.tool +``` + +Example JSON response (fields may vary): +```json +{ + "text": "hello world", + "blocks": [ + {"bbox": [10, 20, 200, 60], "text": "hello", "conf": 87.3}, + {"bbox": [10, 70, 200, 120], "text": "world", "conf": 90.1} + ], + "objects": [ + {"label": "person", "score": 0.63, "bbox": [5, 12, 180, 210]} + ], + "scene_summary": "person x1", + "meta": { + "latency_ms": 123, + "coral_used": true, + "east_used": true, + "ocr_lang": "eng", + "ocr_lang_requested": "eng+rus", + "ocr_lang_strategy": "auto_script", + "ocr_passes": 1, + "request_id": "..." + } +} +``` + +## Notes +- If Coral is absent or fails, the service still returns OCR with `coral_used=false`. +- If EAST model is missing, the whole image is passed to OCR to avoid empty results. +- Keep images under `MAX_IMAGE_MB` (413 returned otherwise). +- Optimize Tesseract languages to those you need to speed up OCR. diff --git a/__pycache__/app.cpython-313.pyc b/__pycache__/app.cpython-313.pyc new file mode 100644 index 0000000..ac7b8de Binary files /dev/null and b/__pycache__/app.cpython-313.pyc differ diff --git a/app.py b/app.py new file mode 100644 index 0000000..fe043a8 --- /dev/null +++ b/app.py @@ -0,0 +1,441 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Lightweight image-to-text API for Jetson Nano + EdgeTPU. +Compatible with Python 3.6. +""" +import argparse +import base64 +import io +import os +import sys +import time +import uuid +import threading +from flask import Flask, request, jsonify + +from core import image_io, coral_vision, text_detect, ocr, schema + +try: + import cv2 +except ImportError: # pragma: no cover + cv2 = None +import numpy as np +from PIL import Image, ImageDraw + +# ---- Configuration ---- + +def env_bool(name, default): + val = os.getenv(name) + if val is None: + return default + return val.lower() in ("1", "true", "yes", "on") + +def env_float(name, default): + val = os.getenv(name) + try: + return float(val) if val is not None else default + except ValueError: + return default + +def env_int(name, default): + val = os.getenv(name) + try: + return int(val) if val is not None else default + except ValueError: + return default + +MODEL_EDGETPU_PATH = os.getenv("MODEL_EDGETPU_PATH", "./models/ssd_mobilenet_v2_coco_quant_postprocess_edgetpu.tflite") +MODEL_LABELS_PATH = os.getenv("MODEL_LABELS_PATH", "./models/coco_labels.txt") +MODEL_EAST_PATH = os.getenv("MODEL_EAST_PATH", "./models/frozen_east_text_detection.pb") +OCR_LANG = os.getenv("OCR_LANG", "eng+rus") +MAX_IMAGE_MB = env_int("MAX_IMAGE_MB", 10) +CONF_THRESHOLD = env_float("CONF_THRESHOLD", 0.4) +RETURN_BOXES = env_bool("RETURN_BOXES", True) +OCR_CONFIG = os.getenv("OCR_CONFIG", "--oem 1 --psm 6") +TESSDATA_PREFIX = os.getenv("TESSDATA_PREFIX") +USE_CUDA_DNN = env_bool("USE_CUDA_DNN", False) +DEBUG_VISUAL = env_bool("DEBUG_VISUAL", False) +SHOW_GUI = env_bool("SHOW_GUI", False) +GUI_TARGET_WIDTH = env_int("GUI_TARGET_WIDTH", 1600) +OCR_UPSCALE = env_float("OCR_UPSCALE", 1.5) # 1.0 = no upscale +OCR_CLAHE = env_bool("OCR_CLAHE", True) +OCR_BILATERAL = env_bool("OCR_BILATERAL", False) +OCR_BLUR = env_int("OCR_BLUR", 0) # 0 disables gaussian blur +OCR_INVERT = env_bool("OCR_INVERT", False) +OCR_USE_THRESHOLD = env_bool("OCR_USE_THRESHOLD", True) +OCR_AUTO_LANG = env_bool("OCR_AUTO_LANG", True) +MAX_CONTENT_LENGTH = MAX_IMAGE_MB * 1024 * 1024 + +app = Flask(__name__) +app.config['MAX_CONTENT_LENGTH'] = MAX_CONTENT_LENGTH + +if not TESSDATA_PREFIX: + # Attempt common system locations + for candidate in ( + "/usr/share/tesseract-ocr/4.00/tessdata_best", + "/usr/share/tesseract-ocr/4.00/tessdata", + ): + if os.path.exists(candidate): + TESSDATA_PREFIX = candidate + break + +if TESSDATA_PREFIX: + os.environ["TESSDATA_PREFIX"] = TESSDATA_PREFIX + +_GUI_FRAME = None +_GUI_LOCK = threading.Lock() +_GUI_THREAD = None + +def _start_gui_thread(): + if not SHOW_GUI or cv2 is None: + return + global _GUI_THREAD + if _GUI_THREAD is not None: + return + def _loop(): + cv2.namedWindow("image2text debug", cv2.WINDOW_NORMAL) + while True: + frame = None + with _GUI_LOCK: + if _GUI_FRAME is not None: + frame = _GUI_FRAME.copy() + if frame is not None: + cv2.imshow("image2text debug", frame) + cv2.resizeWindow("image2text debug", frame.shape[1], frame.shape[0]) + cv2.waitKey(50) + _GUI_THREAD = threading.Thread(target=_loop, daemon=True) + _GUI_THREAD.start() + +def _show_placeholder(): + if not SHOW_GUI or cv2 is None: + return + _gui_show_message("image2text waiting for request") + + +def _gui_show(frame): + if not SHOW_GUI or cv2 is None or frame is None: + return + global _GUI_FRAME + with _GUI_LOCK: + _GUI_FRAME = frame + + +def _gui_show_message(msg): + if not SHOW_GUI or cv2 is None: + return + width = GUI_TARGET_WIDTH if GUI_TARGET_WIDTH > 0 else 1280 + height = int(width * 9 / 16) + canvas = np.ones((height, width, 3), dtype=np.uint8) * 240 + cv2.putText(canvas, msg[:60], (40, height // 2), + cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 0, 0), 2, cv2.LINE_AA) + _gui_show(canvas) + +# ---- Load models once ---- + +coral = coral_vision.CoralVision( + model_path=MODEL_EDGETPU_PATH, + labels_path=MODEL_LABELS_PATH, + score_threshold=CONF_THRESHOLD, +) + +txt_detector = text_detect.TextDetector( + east_model_path=MODEL_EAST_PATH, + score_threshold=CONF_THRESHOLD, + use_cuda=USE_CUDA_DNN, +) + +# ---- Helpers ---- + +def parse_image_from_request(req): + if 'image' in req.files: + data = req.files['image'].read() + source = 'multipart' + else: + try: + payload = req.get_json(force=True, silent=False) + except Exception: + payload = None + if payload and 'image_base64' in payload: + b64_str = payload['image_base64'] or '' + if ',' in b64_str: + b64_str = b64_str.split(',', 1)[1] + try: + data = base64.b64decode(b64_str, validate=False) + except Exception: + return None, 'invalid_base64' + source = 'base64' + else: + return None, 'not_found' + if not data: + return None, 'empty' + if len(data) > MAX_CONTENT_LENGTH: + return None, 'too_large' + return data, source + + +def build_debug_image(img_rgb, boxes, blocks, objects, target_width=1024, return_image=False): + """Return (base64 JPEG, optional ndarray for display) with drawn boxes.""" + try: + vis = img_rgb.copy() + h, w = vis.shape[:2] + scale = 1.0 + if target_width and target_width > 0 and w != target_width: + scale = float(target_width) / float(w) + # avoid crazy upscales + if scale > 3.0: + scale = 3.0 + new_w = int(w * scale) + new_h = int(h * scale) + vis = np.array(Image.fromarray(vis).resize((new_w, new_h))) + # draw using cv2 if available + if cv2 is not None: + vis_bgr = cv2.cvtColor(vis, cv2.COLOR_RGB2BGR) + # text boxes in blue + for b in boxes: + x1, y1, x2, y2 = [int(v * scale) for v in b] + cv2.rectangle(vis_bgr, (x1, y1), (x2, y2), (255, 0, 0), 2) + for blk in blocks: + x1, y1, x2, y2 = [int(v * scale) for v in blk.get("bbox", [0, 0, 0, 0])] + cv2.rectangle(vis_bgr, (x1, y1), (x2, y2), (0, 128, 255), 2) + txt = blk.get("text", "")[:20] + cv2.putText(vis_bgr, txt, (x1, max(0, y1 - 5)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 128, 255), 1, cv2.LINE_AA) + # objects in green + for obj in objects or []: + bbox = obj.get("bbox", [0, 0, 0, 0]) + x1, y1, x2, y2 = [int(v * scale) for v in bbox] + cv2.rectangle(vis_bgr, (x1, y1), (x2, y2), (0, 200, 0), 2) + lbl = obj.get("label", "obj") + cv2.putText(vis_bgr, lbl, (x1, max(0, y1 - 5)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 200, 0), 1, cv2.LINE_AA) + ok, buf = cv2.imencode(".jpg", vis_bgr, [int(cv2.IMWRITE_JPEG_QUALITY), 85]) + if not ok: + return None, None + b64 = base64.b64encode(buf.tobytes()).decode("ascii") + return b64, vis_bgr if return_image else None + # fallback to PIL + img = Image.fromarray(vis) + draw = ImageDraw.Draw(img) + for b in boxes: + x1, y1, x2, y2 = [int(v * scale) for v in b] + draw.rectangle([x1, y1, x2, y2], outline="blue", width=2) + for blk in blocks: + x1, y1, x2, y2 = [int(v * scale) for v in blk.get("bbox", [0, 0, 0, 0])] + draw.rectangle([x1, y1, x2, y2], outline="orange", width=2) + txt = blk.get("text", "")[:20] + draw.text((x1, max(0, y1 - 10)), txt, fill="orange") + for obj in objects or []: + bbox = obj.get("bbox", [0, 0, 0, 0]) + x1, y1, x2, y2 = [int(v * scale) for v in bbox] + draw.rectangle([x1, y1, x2, y2], outline="green", width=2) + draw.text((x1, max(0, y1 - 10)), obj.get("label", "obj"), fill="green") + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=85) + b64 = base64.b64encode(buf.getvalue()).decode("ascii") + return b64, np.array(img) if return_image else None + except Exception: + return None, None + +# ---- Routes ---- + +@app.route('/health', methods=['GET']) +def health(): + return jsonify({"status": "ok", "coral": coral.available}) + + +@app.route('/v1/image2text', methods=['POST']) +def image2text(): + t0 = time.time() + request_id = str(uuid.uuid4()) + data, status = parse_image_from_request(request) + if SHOW_GUI and cv2 is not None: + _gui_show_message("processing request {}".format(request_id[:8])) + + if data is None: + if status == 'too_large': + if SHOW_GUI and cv2 is not None: + _gui_show_message("error: payload too large") + return jsonify({"error": "payload too large"}), 413 + if status in ('not_found', 'empty'): + if SHOW_GUI and cv2 is not None: + _gui_show_message("error: image missing") + return jsonify({"error": "image is required"}), 400 + if SHOW_GUI and cv2 is not None: + _gui_show_message("error: invalid base64") + return jsonify({"error": "invalid image_base64"}), 400 + + img_rgb = image_io.decode_image(data) + if img_rgb is None: + if SHOW_GUI and cv2 is not None: + _gui_show_message("error: unsupported format") + return jsonify({"error": "unsupported image format"}), 415 + + objects = [] + coral_used = False + try: + objects, coral_used = coral.infer(img_rgb) + except Exception as exc: + print("[WARN] Coral failure {}: {}".format(request_id, exc), file=sys.stderr) + objects = [] + coral_used = False + + boxes, east_used = txt_detector.detect(img_rgb) + + prep_options = { + "upscale": OCR_UPSCALE, + "clahe": OCR_CLAHE, + "bilateral": OCR_BILATERAL, + "blur": OCR_BLUR, + "invert": OCR_INVERT, + "use_threshold": OCR_USE_THRESHOLD, + } + + ocr_lang_selected = OCR_LANG + ocr_lang_strategy = "fixed" + if OCR_AUTO_LANG: + try: + ocr_lang_selected, ocr_lang_strategy = ocr.choose_ocr_lang( + img_rgb, + boxes, + requested_lang=OCR_LANG, + prep_options=prep_options, + ) + except Exception as exc: + print("[WARN] OCR lang auto-detect failed {}: {}".format(request_id, exc), file=sys.stderr) + ocr_lang_selected = OCR_LANG + ocr_lang_strategy = "auto_error_fallback" + + text, blocks = ocr.run_ocr( + img_rgb, + boxes, + lang=ocr_lang_selected, + config=OCR_CONFIG, + return_boxes=RETURN_BOXES, + prep_options=prep_options, + ) + ocr_passes = 1 + + # Retry with requested language set when auto-picked language produced no text. + if (not (text or '').strip()) and ocr_lang_selected != OCR_LANG: + retry_text, retry_blocks = ocr.run_ocr( + img_rgb, + boxes, + lang=OCR_LANG, + config=OCR_CONFIG, + return_boxes=RETURN_BOXES, + prep_options=prep_options, + ) + ocr_passes += 1 + if (retry_text or '').strip(): + text, blocks = retry_text, retry_blocks + ocr_lang_selected = OCR_LANG + ocr_lang_strategy = "{}+lang_retry".format(ocr_lang_strategy) + + # Retry with full-frame OCR if EAST boxes were too aggressive and OCR is empty. + if not (text or '').strip(): + h, w = img_rgb.shape[:2] + full_box = (0, 0, w, h) + needs_full_pass = True + if len(boxes) == 1: + try: + cur_box = tuple([int(v) for v in boxes[0]]) + needs_full_pass = cur_box != full_box + except Exception: + needs_full_pass = True + if needs_full_pass: + retry_text, retry_blocks = ocr.run_ocr( + img_rgb, + [full_box], + lang=ocr_lang_selected, + config=OCR_CONFIG, + return_boxes=RETURN_BOXES, + prep_options=prep_options, + ) + ocr_passes += 1 + if (retry_text or '').strip(): + text, blocks = retry_text, retry_blocks + ocr_lang_strategy = "{}+full_frame_retry".format(ocr_lang_strategy) + + scene_summary = schema.summarize_scene(objects) + + latency_ms = int((time.time() - t0) * 1000) + + debug_requested = DEBUG_VISUAL or request.args.get('debug', '').lower() in ('1', 'true', 'yes') + if SHOW_GUI: + debug_requested = True # ensure overlay built when GUI requested + debug_image_b64 = None + debug_image_arr = None + if debug_requested: + target_w = GUI_TARGET_WIDTH if SHOW_GUI else 1024 + debug_image_b64, debug_image_arr = build_debug_image( + img_rgb, + boxes, + blocks if RETURN_BOXES else [], + objects, + target_width=target_w, + return_image=SHOW_GUI, + ) + if SHOW_GUI and cv2 is not None and debug_image_arr is not None: + try: + _gui_show(debug_image_arr) + except Exception: + pass + + resp = schema.build_response( + text=text, + blocks=blocks, + objects=objects, + scene_summary=scene_summary, + metadata={ + "latency_ms": latency_ms, + "coral_used": coral_used, + "east_used": east_used, + "ocr_lang": ocr_lang_selected, + "ocr_lang_requested": OCR_LANG, + "ocr_lang_strategy": ocr_lang_strategy, + "ocr_passes": ocr_passes, + "request_id": request_id, + "debug": debug_requested, + }, + return_boxes=RETURN_BOXES, + ) + + if debug_image_b64: + resp["debug_image_base64"] = debug_image_b64 + + print("[INFO] req={} size={}KB coral={} east={} latency={}ms".format( + request_id, + len(data) // 1024, + coral_used, + east_used, + latency_ms + )) + + return jsonify(resp) + + +# ---- Entrypoint ---- + + +def main(): + parser = argparse.ArgumentParser(description='Image to text API') + parser.add_argument('--host', default='0.0.0.0') + parser.add_argument('--port', type=int, default=8000) + parser.add_argument('--debug-visual', action='store_true', help='force returning debug overlay base64') + parser.add_argument('--show-gui', action='store_true', help='show overlay in a window (requires DISPLAY and OpenCV GUI); forces debug overlay creation') + args = parser.parse_args() + + global DEBUG_VISUAL + if args.debug_visual: + DEBUG_VISUAL = True + global SHOW_GUI + if args.show_gui: + SHOW_GUI = True + if SHOW_GUI: + _start_gui_thread() + _show_placeholder() + + app.run(host=args.host, port=args.port, threaded=True) + + +if __name__ == '__main__': + main() diff --git a/core/__init__.py b/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/__pycache__/image_io.cpython-313.pyc b/core/__pycache__/image_io.cpython-313.pyc new file mode 100644 index 0000000..94120b7 Binary files /dev/null and b/core/__pycache__/image_io.cpython-313.pyc differ diff --git a/core/__pycache__/ocr.cpython-313.pyc b/core/__pycache__/ocr.cpython-313.pyc new file mode 100644 index 0000000..3292b8b Binary files /dev/null and b/core/__pycache__/ocr.cpython-313.pyc differ diff --git a/core/__pycache__/schema.cpython-313.pyc b/core/__pycache__/schema.cpython-313.pyc new file mode 100644 index 0000000..43de3f8 Binary files /dev/null and b/core/__pycache__/schema.cpython-313.pyc differ diff --git a/core/__pycache__/text_detect.cpython-313.pyc b/core/__pycache__/text_detect.cpython-313.pyc new file mode 100644 index 0000000..641683a Binary files /dev/null and b/core/__pycache__/text_detect.cpython-313.pyc differ diff --git a/core/coral_vision.py b/core/coral_vision.py new file mode 100644 index 0000000..6132580 --- /dev/null +++ b/core/coral_vision.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +"""Minimal wrapper around Coral EdgeTPU detection/classification.""" +import os +import sys + +try: + from pycoral.utils.edgetpu import make_interpreter + from pycoral.adapters import common, detect +except ImportError: # pragma: no cover + make_interpreter = None + common = None + detect = None + +import numpy as np +from PIL import Image + + +class CoralVision(object): + def __init__(self, model_path, labels_path, score_threshold=0.4): + self.model_path = model_path + self.labels_path = labels_path + self.score_threshold = score_threshold + self.available = False + self.labels = {} + self.interpreter = None + self._load() + + def _load_labels(self): + labels = {} + if not os.path.exists(self.labels_path): + return labels + with open(self.labels_path, 'r') as f: + for line in f: + if not line.strip(): + continue + pair = line.strip().split(None, 1) + if len(pair) == 2 and pair[0].isdigit(): + labels[int(pair[0])] = pair[1] + else: + # fallback for plain list + labels[len(labels)] = pair[0] + return labels + + def _load(self): + if make_interpreter is None: + print('[WARN] pycoral not installed; Coral disabled', file=sys.stderr) + return + if not os.path.exists(self.model_path): + print('[WARN] Coral model not found at {} - EdgeTPU disabled'.format(self.model_path), file=sys.stderr) + return + try: + self.interpreter = make_interpreter(self.model_path) + self.interpreter.allocate_tensors() + self.labels = self._load_labels() + self.available = True + print('[INFO] Coral model loaded: {}'.format(self.model_path)) + except Exception as exc: + print('[WARN] Failed to init Coral: {}'.format(exc), file=sys.stderr) + self.available = False + + def infer(self, img_rgb): + if not self.available: + return [], False + + h, w, _ = img_rgb.shape + input_size = common.input_size(self.interpreter) + resized = _resize_keep_aspect(img_rgb, input_size) + common.set_input(self.interpreter, resized) + self.interpreter.invoke() + objs = detect.get_objects(self.interpreter, score_threshold=self.score_threshold) + + results = [] + for obj in objs: + bbox = obj.bbox + scale_x = float(w) / float(input_size[0]) + scale_y = float(h) / float(input_size[1]) + x1 = int(bbox.xmin * scale_x) + y1 = int(bbox.ymin * scale_y) + x2 = int(bbox.xmax * scale_x) + y2 = int(bbox.ymax * scale_y) + label = self.labels.get(obj.id, str(obj.id)) + results.append({ + "label": label, + "score": float(obj.score), + "bbox": [x1, y1, x2, y2], + }) + return results, True + + +def _resize_keep_aspect(img_rgb, target_size): + tw, th = target_size + return np.array(Image.fromarray(img_rgb).resize((tw, th))) diff --git a/core/image_io.py b/core/image_io.py new file mode 100644 index 0000000..af0c5ba --- /dev/null +++ b/core/image_io.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +"""Image decoding and preprocessing helpers.""" +import io +import numpy as np +from PIL import Image, ImageSequence + +try: + import cv2 +except ImportError: # pragma: no cover + cv2 = None + + +def decode_image(data): + """Decode bytes to RGB numpy array. Returns None on failure.""" + img = _decode_with_pillow(data) + if img is None and cv2 is not None: + img = _decode_with_cv2(data) + return img + + +def _decode_with_pillow(data): + try: + with Image.open(io.BytesIO(data)) as im: + # pick first frame for GIFs + frame = next(ImageSequence.Iterator(im)) + rgb = frame.convert('RGB') + arr = np.array(rgb) + return arr + except Exception: + return None + + +def _decode_with_cv2(data): + try: + arr = np.frombuffer(data, dtype=np.uint8) + img = cv2.imdecode(arr, cv2.IMREAD_COLOR) + if img is None: + return None + rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + return rgb + except Exception: + return None + + +def prepare_for_ocr(img_rgb, use_threshold=True, upscale=1.0, clahe=False, + bilateral=False, blur=0, invert=False): + """Convert RGB to grayscale and apply light preprocessing for OCR.""" + if img_rgb is None: + return None + + gray = _to_gray(img_rgb) + + # optional upscale (cap at 3x to avoid memory blow-up) + if upscale and upscale > 1.0: + factor = min(float(upscale), 3.0) + h, w = gray.shape[:2] + new_w = int(w * factor) + new_h = int(h * factor) + if cv2 is not None: + gray = cv2.resize(gray, (new_w, new_h), interpolation=cv2.INTER_CUBIC) + else: + gray = np.array(Image.fromarray(gray).resize((new_w, new_h))) + + if clahe and cv2 is not None: + try: + clahe_obj = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8)) + gray = clahe_obj.apply(gray) + except Exception: + pass + + if bilateral and cv2 is not None: + try: + gray = cv2.bilateralFilter(gray, d=5, sigmaColor=75, sigmaSpace=75) + except Exception: + pass + + if blur and blur > 0 and cv2 is not None: + k = int(blur) if int(blur) % 2 == 1 else int(blur) + 1 + try: + gray = cv2.GaussianBlur(gray, (k, k), 0) + except Exception: + pass + + if use_threshold and cv2 is not None: + try: + gray = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, + cv2.THRESH_BINARY, 25, 15) + except Exception: + pass + + if invert: + gray = 255 - gray + + return gray + + +def _to_gray(img_rgb): + if img_rgb.ndim == 3 and img_rgb.shape[2] == 3: + return np.dot(img_rgb[..., :3], [0.2989, 0.5870, 0.1140]).astype('uint8') + return img_rgb.astype('uint8') + + +def crop(img_rgb, bbox): + x1, y1, x2, y2 = bbox + h, w = img_rgb.shape[:2] + x1 = max(0, min(w - 1, int(x1))) + y1 = max(0, min(h - 1, int(y1))) + x2 = max(0, min(w, int(x2))) + y2 = max(0, min(h, int(y2))) + return img_rgb[y1:y2, x1:x2] diff --git a/core/ocr.py b/core/ocr.py new file mode 100644 index 0000000..ac30806 --- /dev/null +++ b/core/ocr.py @@ -0,0 +1,174 @@ +# -*- coding: utf-8 -*- +"""OCR wrapper using pytesseract.""" +import pytesseract +import numpy as np + +from . import image_io + +_AUTO_LANGS = frozenset(("eng", "rus")) +_SCRIPT_TO_LANG = { + "latin": "eng", + "cyrillic": "rus", +} + + +def choose_ocr_lang(img_rgb, boxes, requested_lang='eng+rus', prep_options=None): + """ + Pick OCR language for RU/EN inputs. + + Uses a fast Tesseract OSD pass to detect script (Latin/Cyrillic) on a few + largest text regions. Falls back to requested_lang when uncertain. + """ + requested = (requested_lang or 'eng').strip() + langs = _split_langs(requested) + if set(langs) != _AUTO_LANGS: + return requested, "fixed" + + detected = _detect_lang_via_osd(img_rgb, boxes, prep_options=prep_options) + if detected: + return detected, "auto_script" + return requested, "auto_fallback" + + +def run_ocr(img_rgb, boxes, lang='eng', config='', return_boxes=True, prep_options=None): + if not boxes: + boxes = [(0, 0, img_rgb.shape[1], img_rgb.shape[0])] + + blocks = [] + collected_text = [] + prep_options = prep_options or {} + + for bbox in boxes: + crop = image_io.crop(img_rgb, bbox) + if crop.size == 0: + continue + prepared = image_io.prepare_for_ocr( + crop, + use_threshold=prep_options.get("use_threshold", True), + upscale=prep_options.get("upscale", 1.0), + clahe=prep_options.get("clahe", False), + bilateral=prep_options.get("bilateral", False), + blur=prep_options.get("blur", 0), + invert=prep_options.get("invert", False), + ) + try: + data = pytesseract.image_to_data( + prepared, + lang=lang, + config=config, + output_type=pytesseract.Output.DICT, + ) + texts = [] + confs = [] + for txt, conf in zip(data.get('text', []), data.get('conf', [])): + if txt and txt.strip(): + texts.append(txt.strip()) + try: + confs.append(float(conf)) + except Exception: + confs.append(-1.0) + block_text = ' '.join(texts).strip() + avg_conf = float(np.mean(confs)) if confs else -1.0 + except Exception: + # Fallback to simple string extraction + block_text = pytesseract.image_to_string(prepared, lang=lang, config=config) + avg_conf = -1.0 + + if block_text: + collected_text.append(block_text) + if return_boxes: + blocks.append({ + "bbox": [int(bbox[0]), int(bbox[1]), int(bbox[2]), int(bbox[3])], + "text": block_text, + "conf": avg_conf, + }) + + combined_text = '\n'.join(collected_text) + if not return_boxes: + blocks = [] + return combined_text, blocks + + +def _split_langs(lang): + parts = [] + for item in (lang or '').split('+'): + item = item.strip() + if item and item not in parts: + parts.append(item) + return parts + + +def _bbox_area(bbox): + try: + x1, y1, x2, y2 = bbox + return max(0, int(x2) - int(x1)) * max(0, int(y2) - int(y1)) + except Exception: + return 0 + + +def _probe_boxes(img_rgb, boxes, limit=3): + if not boxes: + h, w = img_rgb.shape[:2] + return [(0, 0, w, h)] + ranked = sorted(list(boxes), key=_bbox_area, reverse=True) + return ranked[:limit] + + +def _prepare_probe_crop(crop, prep_options): + prep_options = prep_options or {} + upscale = prep_options.get("upscale", 1.0) + try: + upscale = float(upscale) + except Exception: + upscale = 1.0 + if upscale < 1.0: + upscale = 1.0 + if upscale > 1.5: + upscale = 1.5 + + return image_io.prepare_for_ocr( + crop, + use_threshold=False, + upscale=upscale, + clahe=prep_options.get("clahe", False), + bilateral=False, + blur=0, + invert=False, + ) + + +def _detect_lang_via_osd(img_rgb, boxes, prep_options=None): + votes = {"eng": 0, "rus": 0} + for bbox in _probe_boxes(img_rgb, boxes): + crop = image_io.crop(img_rgb, bbox) + if crop is None or crop.size == 0: + continue + prepared = _prepare_probe_crop(crop, prep_options) + if prepared is None or prepared.size == 0: + continue + lang = _lang_from_osd(prepared) + if lang in votes: + votes[lang] += 1 + + if votes["eng"] == votes["rus"]: + return None + if votes["eng"] > votes["rus"]: + return "eng" + if votes["rus"] > votes["eng"]: + return "rus" + return None + + +def _lang_from_osd(prepared): + try: + osd_text = pytesseract.image_to_osd(prepared, config='--psm 0') + except Exception: + return None + + for line in osd_text.splitlines(): + line_low = line.lower() + if not line_low.startswith("script:"): + continue + script = line.split(':', 1)[1].strip().lower() + return _SCRIPT_TO_LANG.get(script) + return None diff --git a/core/schema.py b/core/schema.py new file mode 100644 index 0000000..2927567 --- /dev/null +++ b/core/schema.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +"""Response assembly utilities.""" +from collections import Counter + + +def summarize_scene(objects): + if not objects: + return "no objects detected" + labels = [o.get('label', 'unknown') for o in objects] + counts = Counter(labels) + top = counts.most_common(3) + parts = ["{} x{}".format(lbl, cnt) for lbl, cnt in top] + return ', '.join(parts) + + +def build_response(text, blocks, objects, scene_summary, metadata, return_boxes=True): + resp = { + "text": text or "", + "blocks": blocks if return_boxes else [], + "objects": objects or [], + "scene_summary": scene_summary or "", + "meta": metadata or {}, + } + return resp diff --git a/core/text_detect.py b/core/text_detect.py new file mode 100644 index 0000000..17ab3a1 --- /dev/null +++ b/core/text_detect.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +"""Text region detection using OpenCV EAST with graceful fallback.""" +import os +import sys +import numpy as np + +try: + import cv2 +except ImportError: # pragma: no cover + cv2 = None + + +class TextDetector(object): + def __init__(self, east_model_path, score_threshold=0.4, use_cuda=False): + self.east_model_path = east_model_path + self.score_threshold = score_threshold + self.use_cuda = use_cuda + self.net = None + self.east_available = False + self._load() + + def _load(self): + if cv2 is None: + print('[WARN] OpenCV not installed; EAST disabled', file=sys.stderr) + return + if not os.path.exists(self.east_model_path): + print('[INFO] EAST model not found; will use whole-image fallback', file=sys.stderr) + return + try: + self.net = cv2.dnn.readNet(self.east_model_path) + if self.use_cuda and hasattr(cv2, "cuda") and cv2.cuda.getCudaEnabledDeviceCount() > 0: + try: + self.net.setPreferableBackend(cv2.dnn.DNN_BACKEND_CUDA) + self.net.setPreferableTarget(cv2.dnn.DNN_TARGET_CUDA) + print('[INFO] EAST using CUDA backend/target', file=sys.stderr) + except Exception as exc: + print('[WARN] Failed to enable CUDA for EAST: {}'.format(exc), file=sys.stderr) + self.east_available = True + print('[INFO] EAST model loaded: {}'.format(self.east_model_path)) + except Exception as exc: + print('[WARN] Failed to load EAST model: {}'.format(exc), file=sys.stderr) + self.east_available = False + + def detect(self, img_rgb): + if not self.east_available: + h, w = img_rgb.shape[:2] + return [(_int0(0), _int0(0), _int0(w), _int0(h))], False + + h, w = img_rgb.shape[:2] + # EAST expects width/height divisible by 32 + new_w = 320 + new_h = 320 + blob = cv2.dnn.blobFromImage(img_rgb, 1.0, (new_w, new_h), + (123.68, 116.78, 103.94), swapRB=True, crop=False) + self.net.setInput(blob) + scores, geometry = self.net.forward([ + "feature_fusion/Conv_7/Sigmoid", + "feature_fusion/concat_3" + ]) + rectangles, confidences = self._decode(scores, geometry, self.score_threshold) + indices = cv2.dnn.NMSBoxes(rectangles, confidences, self.score_threshold, 0.4) + + boxes = [] + rW = float(w) / float(new_w) + rH = float(h) / float(new_h) + if len(indices) > 0: + for i in indices.flatten(): + x, y, bw, bh = rectangles[i] + x1 = int(x * rW) + y1 = int(y * rH) + x2 = int((x + bw) * rW) + y2 = int((y + bh) * rH) + boxes.append((_int0(x1), _int0(y1), _int0(x2), _int0(y2))) + if not boxes: + boxes.append((_int0(0), _int0(0), _int0(w), _int0(h))) + return boxes, True + + def _decode(self, scores, geometry, score_thresh): + num_rows, num_cols = scores.shape[2:4] + rectangles = [] + confidences = [] + for y in range(num_rows): + scores_data = scores[0, 0, y] + x0 = geometry[0, 0, y] + x1 = geometry[0, 1, y] + x2 = geometry[0, 2, y] + x3 = geometry[0, 3, y] + angles = geometry[0, 4, y] + for x in range(num_cols): + score = scores_data[x] + if score < score_thresh: + continue + offset_x = x * 4.0 + offset_y = y * 4.0 + angle = angles[x] + cos = np.cos(angle) + sin = np.sin(angle) + h = x0[x] + x2[x] + w = x1[x] + x3[x] + end_x = int(offset_x + (cos * x1[x]) + (sin * x2[x])) + end_y = int(offset_y - (sin * x1[x]) + (cos * x2[x])) + start_x = int(end_x - w) + start_y = int(end_y - h) + rectangles.append((start_x, start_y, int(w), int(h))) + confidences.append(float(score)) + return rectangles, confidences + + +def _int0(val): + try: + return int(val) + except Exception: + return 0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..891666e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +Flask==2.0.3 +waitress==1.4.4 +Pillow==8.4.0 +numpy==1.19.5 +pytesseract==0.3.10 +pycoral==0.1.1 +opencv-python==4.5.3.56 diff --git a/run_service.sh b/run_service.sh new file mode 100755 index 0000000..4d6a5e2 --- /dev/null +++ b/run_service.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Runner for image2text service on Jetson Nano +# Adjust env vars below as needed. + +set -euo pipefail + +# ---- Configurable env ---- +export MODEL_EDGETPU_PATH=${MODEL_EDGETPU_PATH:-./models/ssd_mobilenet_v2_coco_quant_postprocess_edgetpu.tflite} +export MODEL_LABELS_PATH=${MODEL_LABELS_PATH:-./models/coco_labels.txt} +export MODEL_EAST_PATH=${MODEL_EAST_PATH:-./models/frozen_east_text_detection.pb} +export OCR_LANG=${OCR_LANG:-eng+rus} +export OCR_CONFIG=${OCR_CONFIG:---oem 1 --psm 6} +# Point to tessdata_best if installed +export TESSDATA_PREFIX=${TESSDATA_PREFIX:-} +export MAX_IMAGE_MB=${MAX_IMAGE_MB:-10} +export CONF_THRESHOLD=${CONF_THRESHOLD:-0.4} +export RETURN_BOXES=${RETURN_BOXES:-true} +# Enable CUDA DNN for EAST if your OpenCV is built with CUDA +export USE_CUDA_DNN=${USE_CUDA_DNN:-false} + +HOST=${HOST:-0.0.0.0} +PORT=${PORT:-8000} + +python3 app.py --host "$HOST" --port "$PORT"