diff --git a/database/schema.py b/database/schema.py index 82b7e80..52a4d57 100644 --- a/database/schema.py +++ b/database/schema.py @@ -16,11 +16,16 @@ def _migrate(conn): ("registry_office", "TEXT"), ("owns_properties", "BOOLEAN"), ("declared_property_count", "INTEGER"), + ("image_hash", "TEXT"), + ("duplicate_of", "INTEGER"), ] for col, col_type in migrations: if col not in existing: conn.execute(f"ALTER TABLE documents ADD COLUMN {col} {col_type}") + conn.execute("CREATE INDEX IF NOT EXISTS idx_documents_image_hash ON documents(image_hash)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_documents_request_number ON documents(request_number)") + cursor = conn.execute("PRAGMA table_info(properties)") existing_prop = {row[1] for row in cursor.fetchall()} migrations_prop = [ diff --git a/main.py b/main.py index f69d6f9..4c4bf1b 100644 --- a/main.py +++ b/main.py @@ -3,11 +3,27 @@ from pathlib import Path from fastapi import FastAPI, Request from fastapi.responses import RedirectResponse from fastapi.staticfiles import StaticFiles +from starlette.types import Scope from config import UPLOAD_DIR, ENVIRONMENT from database.schema import create_tables from routers import documents, review, search, upload, auth + +class CachedStaticFiles(StaticFiles): + """StaticFiles that sets a long Cache-Control header so browsers + don't re-download the same image on every refresh.""" + + def __init__(self, *args, max_age: int = 86400, **kwargs): + super().__init__(*args, **kwargs) + self._max_age = max_age + + async def get_response(self, path: str, scope: Scope): + response = await super().get_response(path, scope) + if response.status_code == 200: + response.headers["Cache-Control"] = f"public, max-age={self._max_age}" + return response + # Disable docs/openapi in production if ENVIRONMENT == "production": app = FastAPI(title="Lebanese Real Estate Registry", docs_url=None, redoc_url=None, openapi_url=None) @@ -39,8 +55,8 @@ async def startup(): # A script should be used for initial setup. -app.mount("/uploads", StaticFiles(directory=UPLOAD_DIR), name="uploads") -app.mount("/static", StaticFiles(directory="static"), name="static") +app.mount("/uploads", CachedStaticFiles(directory=UPLOAD_DIR, max_age=604800), name="uploads") +app.mount("/static", CachedStaticFiles(directory="static", max_age=86400), name="static") app.include_router(upload.router) app.include_router(review.router) diff --git a/routers/documents.py b/routers/documents.py index 2a60b28..b99a1a1 100644 --- a/routers/documents.py +++ b/routers/documents.py @@ -2,35 +2,93 @@ import os from pathlib import Path from fastapi import APIRouter, Request -from fastapi.responses import JSONResponse +from fastapi.responses import FileResponse, JSONResponse from fastapi.templating import Jinja2Templates from config import UPLOAD_DIR from database.connection import get_db +from services.thumbnails import get_or_create_thumbnail router = APIRouter() templates = Jinja2Templates(directory="templates") +@router.get("/thumbs/{rel_path:path}") +async def serve_thumbnail(rel_path: str, size: int = 320): + """Serve a cached JPEG thumbnail for an uploaded image. + Falls back to the original file if thumbnail generation fails.""" + # Clamp size to sensible values to prevent DoS + if size not in (160, 240, 320, 480, 640): + size = 320 + + thumb = get_or_create_thumbnail(rel_path, size=size) + if thumb is None: + # Fallback: serve original if it exists + base = Path(UPLOAD_DIR).resolve() + full = (base / rel_path).resolve() + try: + full.relative_to(base) + except ValueError: + return JSONResponse({"error": "not found"}, status_code=404) + if not full.is_file(): + return JSONResponse({"error": "not found"}, status_code=404) + return FileResponse( + full, + headers={"Cache-Control": "public, max-age=604800"}, + ) + + return FileResponse( + thumb, + media_type="image/jpeg", + headers={"Cache-Control": "public, max-age=604800, immutable"}, + ) + +_DOC_LIST_COLUMNS = ( + "d.id, d.person_id, d.image_path, d.request_number, d.status, " + "d.duplicate_of, d.extraction_error, d.created_at" +) + +PAGE_SIZE = 50 + + @router.get("/documents") -async def document_queue(request: Request, status: str = "", uploaded: int = 0): +async def document_queue( + request: Request, + status: str = "", + uploaded: int = 0, + duplicates: int = 0, + page: int = 1, +): + if page < 1: + page = 1 + offset = (page - 1) * PAGE_SIZE + with get_db() as conn: if status: rows = conn.execute( - """SELECT d.*, p.first_name, p.family_name + f"""SELECT {_DOC_LIST_COLUMNS}, p.first_name, p.family_name FROM documents d LEFT JOIN persons p ON p.id = d.person_id WHERE d.status=? - ORDER BY d.id DESC""", - (status,), + ORDER BY d.id DESC + LIMIT ? OFFSET ?""", + (status, PAGE_SIZE, offset), ).fetchall() + total_filtered = conn.execute( + "SELECT COUNT(*) AS n FROM documents WHERE status=?", (status,) + ).fetchone()["n"] else: rows = conn.execute( - """SELECT d.*, p.first_name, p.family_name + f"""SELECT {_DOC_LIST_COLUMNS}, p.first_name, p.family_name FROM documents d LEFT JOIN persons p ON p.id = d.person_id - ORDER BY d.id DESC""" + ORDER BY d.id DESC + LIMIT ? OFFSET ?""", + (PAGE_SIZE, offset), ).fetchall() + total_filtered = conn.execute( + "SELECT COUNT(*) AS n FROM documents" + ).fetchone()["n"] stats = conn.execute( """SELECT @@ -38,10 +96,13 @@ async def document_queue(request: Request, status: str = "", uploaded: int = 0): SUM(CASE WHEN status='confirmed' THEN 1 ELSE 0 END) AS confirmed, SUM(CASE WHEN status='extracted' THEN 1 ELSE 0 END) AS pending_review, SUM(CASE WHEN status='pending' THEN 1 ELSE 0 END) AS processing, - SUM(CASE WHEN status='error' THEN 1 ELSE 0 END) AS errors + SUM(CASE WHEN status='error' THEN 1 ELSE 0 END) AS errors, + SUM(CASE WHEN duplicate_of IS NOT NULL THEN 1 ELSE 0 END) AS duplicates FROM documents WHERE status != 'staged'""" ).fetchone() + total_pages = max(1, (total_filtered + PAGE_SIZE - 1) // PAGE_SIZE) + return templates.TemplateResponse( request, "documents.html", @@ -50,6 +111,11 @@ async def document_queue(request: Request, status: str = "", uploaded: int = 0): "stats": dict(stats) if stats else {}, "current_status": status, "uploaded": uploaded, + "duplicates_skipped": duplicates, + "page": page, + "total_pages": total_pages, + "page_size": PAGE_SIZE, + "total_filtered": total_filtered, }, ) @@ -88,3 +154,196 @@ async def delete_document(doc_id: int): pass return JSONResponse({"ok": True}) + + +@router.post("/documents/{doc_id}/retry") +async def retry_document(doc_id: int): + """Re-run AI extraction on an errored or extracted document.""" + import asyncio + from services.extractor import get_default_provider + from routers.upload import _extract_and_save + + with get_db() as conn: + doc = conn.execute( + "SELECT id, image_path, provider, status FROM documents WHERE id=?", + (doc_id,), + ).fetchone() + if not doc: + return JSONResponse({"error": "not found"}, status_code=404) + + # Clear previous extraction data and requeue + conn.execute("DELETE FROM properties WHERE document_id=?", (doc_id,)) + conn.execute( + """UPDATE documents SET status='pending', + extraction_error=NULL, + raw_extraction_json=NULL, + updated_at=CURRENT_TIMESTAMP + WHERE id=?""", + (doc_id,), + ) + + provider = doc["provider"] or get_default_provider() + asyncio.create_task(_extract_and_save(doc_id, doc["image_path"], provider)) + return JSONResponse({"ok": True, "status": "pending"}) + + +@router.post("/documents/retry-errors") +async def retry_all_errors(): + """Re-run extraction for every document currently in error state.""" + import asyncio + from services.extractor import get_default_provider + from routers.upload import _extract_and_save + + default_provider = get_default_provider() + with get_db() as conn: + rows = conn.execute( + "SELECT id, image_path, provider FROM documents WHERE status='error'" + ).fetchall() + ids = [r["id"] for r in rows] + if ids: + conn.execute("DELETE FROM properties WHERE document_id IN (" + ",".join("?" * len(ids)) + ")", ids) + conn.execute( + "UPDATE documents SET status='pending', extraction_error=NULL, raw_extraction_json=NULL, updated_at=CURRENT_TIMESTAMP WHERE id IN (" + ",".join("?" * len(ids)) + ")", + ids, + ) + + for row in rows: + provider = row["provider"] or default_provider + asyncio.create_task(_extract_and_save(row["id"], row["image_path"], provider)) + + return JSONResponse({"ok": True, "retried": len(rows)}) + + +@router.post("/documents/scan-duplicates") +async def scan_duplicates(): + """ + Backfill image_hash for existing documents and flag duplicates. + A doc is marked duplicate_of the earliest document (lowest id) that shares + either the exact image hash OR the same request_number + search_scope + page_info. + """ + import hashlib + + def _hash_file(path: Path) -> str | None: + try: + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + except OSError: + return None + + hashed = 0 + flagged_by_hash = 0 + flagged_by_request = 0 + + with get_db() as conn: + # 1. Hash any document missing image_hash + rows = conn.execute( + "SELECT id, image_path FROM documents WHERE image_hash IS NULL OR image_hash=''" + ).fetchall() + for row in rows: + full_path = Path(UPLOAD_DIR) / row["image_path"] + digest = _hash_file(full_path) + if digest: + conn.execute( + "UPDATE documents SET image_hash=? WHERE id=?", + (digest, row["id"]), + ) + hashed += 1 + + # 2. Flag duplicates by image_hash (keep earliest) + hash_groups = conn.execute( + """SELECT image_hash, MIN(id) AS keeper, COUNT(*) AS n + FROM documents + WHERE image_hash IS NOT NULL AND image_hash != '' + AND status != 'staged' + GROUP BY image_hash + HAVING n > 1""" + ).fetchall() + for g in hash_groups: + result = conn.execute( + """UPDATE documents SET duplicate_of=? + WHERE image_hash=? AND id != ? AND status != 'staged'""", + (g["keeper"], g["image_hash"], g["keeper"]), + ) + flagged_by_hash += result.rowcount or 0 + + # 3. Flag duplicates by request_number + search_scope + page_info (keep earliest) + logical_groups = conn.execute( + """SELECT TRIM(request_number) AS rn, + COALESCE(TRIM(search_scope),'') AS sc, + COALESCE(TRIM(page_info),'') AS pi, + MIN(id) AS keeper, + COUNT(*) AS n + FROM documents + WHERE status IN ('extracted','confirmed') + AND request_number IS NOT NULL AND TRIM(request_number) != '' + GROUP BY rn, sc, pi + HAVING n > 1""" + ).fetchall() + for g in logical_groups: + result = conn.execute( + """UPDATE documents SET duplicate_of=? + WHERE TRIM(request_number)=? + AND COALESCE(TRIM(search_scope),'')=? + AND COALESCE(TRIM(page_info),'')=? + AND id != ? + AND duplicate_of IS NULL + AND status IN ('extracted','confirmed')""", + (g["keeper"], g["rn"], g["sc"], g["pi"], g["keeper"]), + ) + flagged_by_request += result.rowcount or 0 + + total_dupes = conn.execute( + "SELECT COUNT(*) AS n FROM documents WHERE duplicate_of IS NOT NULL" + ).fetchone()["n"] + + return JSONResponse({ + "ok": True, + "hashed": hashed, + "flagged_by_hash": flagged_by_hash, + "flagged_by_request": flagged_by_request, + "total_duplicates": total_dupes, + }) + + +@router.get("/documents/duplicates") +async def duplicates_view(request: Request): + """List all documents flagged as duplicates alongside their originals.""" + with get_db() as conn: + rows = conn.execute( + """SELECT d.*, p.first_name, p.family_name, + o.request_number AS orig_request_number, + o.created_at AS orig_created_at + FROM documents d + LEFT JOIN persons p ON p.id = d.person_id + LEFT JOIN documents o ON o.id = d.duplicate_of + WHERE d.duplicate_of IS NOT NULL + ORDER BY d.duplicate_of, d.id""" + ).fetchall() + return templates.TemplateResponse( + request, + "duplicates.html", + {"documents": [dict(r) for r in rows]}, + ) + + +@router.post("/documents/delete-duplicates") +async def delete_all_duplicates(): + """Delete every document flagged as duplicate_of another doc.""" + deleted = 0 + with get_db() as conn: + rows = conn.execute( + "SELECT id, image_path FROM documents WHERE duplicate_of IS NOT NULL" + ).fetchall() + for row in rows: + conn.execute("DELETE FROM properties WHERE document_id=?", (row["id"],)) + conn.execute("DELETE FROM documents WHERE id=?", (row["id"],)) + # Remove file from disk + try: + (Path(UPLOAD_DIR) / row["image_path"]).unlink(missing_ok=True) + except OSError: + pass + deleted += 1 + return JSONResponse({"ok": True, "deleted": deleted}) diff --git a/routers/upload.py b/routers/upload.py index 911def1..3fe50a8 100644 --- a/routers/upload.py +++ b/routers/upload.py @@ -1,11 +1,12 @@ import asyncio +import hashlib import json import uuid from datetime import date from pathlib import Path from fastapi import APIRouter, File, Form, Request, UploadFile -from fastapi.responses import RedirectResponse +from fastapi.responses import JSONResponse, RedirectResponse from fastapi.templating import Jinja2Templates from config import UPLOAD_DIR @@ -21,6 +22,30 @@ ALLOWED_PDF_EXTS = {".pdf"} ALLOWED_EXTENSIONS = ALLOWED_IMAGE_EXTS | ALLOWED_PDF_EXTS +def _hash_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _hash_file(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def _find_duplicate(conn, image_hash: str) -> dict | None: + """Return an existing non-staged document sharing the same image hash.""" + row = conn.execute( + """SELECT id, status, image_path, person_id, request_number + FROM documents + WHERE image_hash=? AND status != 'staged' + ORDER BY id LIMIT 1""", + (image_hash,), + ).fetchone() + return dict(row) if row else None + + def _save_image(file_bytes: bytes, original_name: str) -> str: """Save image to uploads/{date}/{uuid}_{name} and return relative path.""" today = date.today().isoformat() @@ -90,6 +115,26 @@ async def _extract_and_save(doc_id: int, image_path: str, provider: str = ""): ), ) + # Flag logical duplicates: same request_number + search_scope + page_number + req_num = (data.get("request_number") or "").strip() + scope = (data.get("search_scope") or "").strip() + page_info = (data.get("page_info") or "").strip() + if req_num: + existing = conn.execute( + """SELECT id FROM documents + WHERE id != ? AND request_number=? + AND COALESCE(search_scope,'')=? + AND COALESCE(page_info,'')=? + AND status IN ('extracted','confirmed') + ORDER BY id LIMIT 1""", + (doc_id, req_num, scope, page_info), + ).fetchone() + if existing: + conn.execute( + "UPDATE documents SET duplicate_of=? WHERE id=?", + (existing["id"], doc_id), + ) + except Exception as e: with get_db() as conn: conn.execute( @@ -129,6 +174,7 @@ async def upload_files( provider = get_default_provider() doc_ids = [] + duplicates = [] for upload in files: suffix = Path(upload.filename).suffix.lower() @@ -138,16 +184,32 @@ async def upload_files( file_bytes = await upload.read() if suffix in ALLOWED_PDF_EXTS: - # PDF: split into per-page images + # PDF: split into per-page images; hash each rendered page pages = pdf_to_images(file_bytes, upload.filename) for page_info in pages: + page_path = Path(UPLOAD_DIR) / page_info["image_path"] + page_hash = _hash_file(page_path) with get_db() as conn: + dup = _find_duplicate(conn, page_hash) + if dup: + # Discard the freshly rendered duplicate page + try: + page_path.unlink(missing_ok=True) + except OSError: + pass + duplicates.append({ + "name": f"{upload.filename} (p{page_info['page_number']})", + "existing_id": dup["id"], + "status": dup["status"], + }) + continue cursor = conn.execute( """INSERT INTO documents - (image_path, status, provider, pdf_group_id, page_number) - VALUES (?, 'pending', ?, ?, ?)""", + (image_path, image_hash, status, provider, pdf_group_id, page_number) + VALUES (?, ?, 'pending', ?, ?, ?)""", ( page_info["image_path"], + page_hash, provider, page_info["pdf_group_id"], page_info["page_number"], @@ -156,11 +218,20 @@ async def upload_files( doc_ids.append((cursor.lastrowid, page_info["image_path"])) else: # Image file - rel_path = _save_image(file_bytes, upload.filename) + image_hash = _hash_bytes(file_bytes) with get_db() as conn: + dup = _find_duplicate(conn, image_hash) + if dup: + duplicates.append({ + "name": upload.filename, + "existing_id": dup["id"], + "status": dup["status"], + }) + continue + rel_path = _save_image(file_bytes, upload.filename) cursor = conn.execute( - "INSERT INTO documents (image_path, status, provider) VALUES (?, 'pending', ?)", - (rel_path, provider), + "INSERT INTO documents (image_path, image_hash, status, provider) VALUES (?, ?, 'pending', ?)", + (rel_path, image_hash, provider), ) doc_ids.append((cursor.lastrowid, rel_path)) @@ -168,9 +239,16 @@ async def upload_files( for doc_id, rel_path in doc_ids: asyncio.create_task(_extract_and_save(doc_id, rel_path, provider)) - if len(doc_ids) == 1: + if len(doc_ids) == 1 and not duplicates: return RedirectResponse(f"/review/{doc_ids[0][0]}?wait=1", status_code=303) - return RedirectResponse("/documents?uploaded=1", status_code=303) + + query = "uploaded=1" + if duplicates: + query += f"&duplicates={len(duplicates)}" + # Route user to the first duplicate so they can see which doc matched + if not doc_ids: + return RedirectResponse(f"/review/{duplicates[0]['existing_id']}", status_code=303) + return RedirectResponse(f"/documents?{query}", status_code=303) # ─── Two-step workflow: stage images, then process ───────────── @@ -182,6 +260,7 @@ async def stage_file( ): """Save uploaded images without triggering AI extraction.""" staged = [] + duplicates = [] for upload in files: suffix = Path(upload.filename).suffix.lower() if suffix not in ALLOWED_EXTENSIONS: @@ -192,13 +271,28 @@ async def stage_file( if suffix in ALLOWED_PDF_EXTS: pages = pdf_to_images(file_bytes, upload.filename) for page_info in pages: + page_path = Path(UPLOAD_DIR) / page_info["image_path"] + page_hash = _hash_file(page_path) with get_db() as conn: + dup = _find_duplicate(conn, page_hash) + if dup: + try: + page_path.unlink(missing_ok=True) + except OSError: + pass + duplicates.append({ + "name": f"{upload.filename} (p{page_info['page_number']})", + "existing_id": dup["id"], + "status": dup["status"], + }) + continue cursor = conn.execute( """INSERT INTO documents - (image_path, status, pdf_group_id, page_number) - VALUES (?, 'staged', ?, ?)""", + (image_path, image_hash, status, pdf_group_id, page_number) + VALUES (?, ?, 'staged', ?, ?)""", ( page_info["image_path"], + page_hash, page_info["pdf_group_id"], page_info["page_number"], ), @@ -209,11 +303,20 @@ async def stage_file( "name": f"{upload.filename} (p{page_info['page_number']})", }) else: - rel_path = _save_image(file_bytes, upload.filename) + image_hash = _hash_bytes(file_bytes) with get_db() as conn: + dup = _find_duplicate(conn, image_hash) + if dup: + duplicates.append({ + "name": upload.filename, + "existing_id": dup["id"], + "status": dup["status"], + }) + continue + rel_path = _save_image(file_bytes, upload.filename) cursor = conn.execute( - "INSERT INTO documents (image_path, status) VALUES (?, 'staged')", - (rel_path,), + "INSERT INTO documents (image_path, image_hash, status) VALUES (?, ?, 'staged')", + (rel_path, image_hash), ) staged.append({ "id": cursor.lastrowid, @@ -221,8 +324,7 @@ async def stage_file( "name": upload.filename, }) - from fastapi.responses import JSONResponse - return JSONResponse({"staged": staged}) + return JSONResponse({"staged": staged, "duplicates": duplicates}) @router.post("/upload/process-staged") @@ -259,7 +361,6 @@ async def process_staged( @router.delete("/upload/staged/{doc_id}") async def remove_staged(doc_id: int): """Remove a single staged document before processing.""" - from fastapi.responses import JSONResponse with get_db() as conn: row = conn.execute("SELECT image_path FROM documents WHERE id=? AND status='staged'", (doc_id,)).fetchone() if row: diff --git a/services/extractor.py b/services/extractor.py index daff2be..eda2945 100644 --- a/services/extractor.py +++ b/services/extractor.py @@ -131,8 +131,7 @@ async def verify_page_correlation( } ], ) - raw_text = _strip_code_fences(message.content[0].text) - result = json.loads(raw_text) + result = _parse_json_lenient(message.content[0].text) elif provider == "gemini": import asyncio from google import genai @@ -176,8 +175,7 @@ async def verify_page_correlation( return response.text raw_text = await asyncio.get_event_loop().run_in_executor(None, _call) - raw_text = _strip_code_fences(raw_text) - result = json.loads(raw_text) + result = _parse_json_lenient(raw_text) else: raise ValueError(f"Provider {provider} does not support AI correlation verification") @@ -221,6 +219,36 @@ def _strip_code_fences(text: str) -> str: return text.strip() +def _parse_json_lenient(text: str) -> dict: + """Parse JSON even if the model added prose, trailing commas, or truncation.""" + text = _strip_code_fences(text or "") + if not text: + raise ValueError("Empty response from model") + + # Fast path + try: + return json.loads(text) + except Exception: + pass + + # Extract the first {...} block (greedy to last }) + start = text.find("{") + end = text.rfind("}") + if start != -1 and end > start: + candidate = text[start:end + 1] + try: + return json.loads(candidate) + except Exception: + # Remove trailing commas before } or ] + cleaned = re.sub(r",(\s*[}\]])", r"\1", candidate) + try: + return json.loads(cleaned) + except Exception: + pass + + raise ValueError(f"Could not parse model response as JSON: {text[:200]}") + + def get_available_providers() -> list[dict]: """Return list of all available providers (EasyOCR is always available).""" providers = [ @@ -248,41 +276,55 @@ def get_default_provider() -> str: async def _extract_with_claude(image_path: str) -> dict: import anthropic + import asyncio client = anthropic.AsyncAnthropic(api_key=ANTHROPIC_API_KEY) full_path = _resolve_path(image_path) img_data, media_type = _encode_image(full_path) - message = await client.messages.create( - model=CLAUDE_MODEL, - max_tokens=4096, - system=[ - { - "type": "text", - "text": SYSTEM_PROMPT, - "cache_control": {"type": "ephemeral"}, - } - ], - messages=[ - { - "role": "user", - "content": [ + last_error = None + for attempt in range(3): + try: + message = await client.messages.create( + model=CLAUDE_MODEL, + max_tokens=8192, + system=[ { - "type": "image", - "source": { - "type": "base64", - "media_type": media_type, - "data": img_data, - }, - }, - {"type": "text", "text": USER_PROMPT}, + "type": "text", + "text": SYSTEM_PROMPT, + "cache_control": {"type": "ephemeral"}, + } ], - } - ], - ) - - raw_text = _strip_code_fences(message.content[0].text) - return json.loads(raw_text) + messages=[ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": media_type, + "data": img_data, + }, + }, + {"type": "text", "text": USER_PROMPT}, + ], + } + ], + ) + return _parse_json_lenient(message.content[0].text) + except Exception as e: + last_error = e + err_str = str(e) + # Transient: overload/rate-limit/network — back off and retry + if any(code in err_str for code in ("529", "503", "502", "500", "overload", "Overload", "timeout", "Timeout")): + await asyncio.sleep(2 * (attempt + 1)) + continue + if "429" in err_str and attempt < 2: + await asyncio.sleep(5 * (attempt + 1)) + continue + raise + raise last_error # ─── Gemini extraction ─────────────────────────────────────────── @@ -320,24 +362,36 @@ async def _extract_with_gemini(image_path: str) -> dict: max_output_tokens=8192, ), ) - return response.text + text = response.text + if not text: + try: + parts = [] + for cand in (response.candidates or []): + for part in (cand.content.parts or []): + if getattr(part, "text", None): + parts.append(part.text) + text = "".join(parts) + except Exception: + pass + if not text: + raise ValueError("Gemini returned empty response") + return text last_error = None for model_name in models_to_try: - for attempt in range(2): + for attempt in range(3): try: raw_text = await asyncio.get_event_loop().run_in_executor(None, lambda m=model_name: _call(m)) - raw_text = _strip_code_fences(raw_text) - return json.loads(raw_text) + return _parse_json_lenient(raw_text) except Exception as e: last_error = e err_str = str(e) - # On quota exhaustion, skip to next model immediately (don't waste retries) + # Quota exhaustion — skip to next model immediately if "429" in err_str or "RESOURCE_EXHAUSTED" in err_str: break - # On temporary overload, wait and retry same model - if "503" in err_str or "UNAVAILABLE" in err_str: - await asyncio.sleep(3 * (attempt + 1)) + # Transient errors — back off and retry same model + if any(code in err_str for code in ("503", "502", "500", "UNAVAILABLE", "DEADLINE", "timeout", "Timeout", "empty response", "Could not parse")): + await asyncio.sleep(2 * (attempt + 1)) continue raise diff --git a/services/thumbnails.py b/services/thumbnails.py new file mode 100644 index 0000000..c0bad8e --- /dev/null +++ b/services/thumbnails.py @@ -0,0 +1,61 @@ +"""On-demand thumbnail generation with disk caching. + +Thumbnails are stored under UPLOAD_DIR/.thumbs//.jpg +and generated lazily the first time they are requested. This keeps the +documents list fast even with hundreds of full-resolution uploads. +""" +from __future__ import annotations + +from pathlib import Path +from typing import Optional + +from PIL import Image, ImageOps + +from config import UPLOAD_DIR + +THUMB_ROOT = Path(UPLOAD_DIR) / ".thumbs" +DEFAULT_SIZE = 320 # px on the longest edge +JPEG_QUALITY = 78 + + +def _safe_rel(rel_path: str) -> Optional[Path]: + """Resolve rel_path against UPLOAD_DIR, rejecting traversal.""" + base = Path(UPLOAD_DIR).resolve() + target = (base / rel_path).resolve() + try: + target.relative_to(base) + except ValueError: + return None + return target + + +def get_or_create_thumbnail(rel_path: str, size: int = DEFAULT_SIZE) -> Optional[Path]: + """Return a path to a cached JPEG thumbnail for the given upload, + creating it if needed. Returns None if the source doesn't exist.""" + source = _safe_rel(rel_path) + if source is None or not source.is_file(): + return None + + thumb_dir = THUMB_ROOT / str(size) + thumb_path = thumb_dir / (rel_path + ".jpg") + + # Regenerate if missing or source is newer + if thumb_path.is_file(): + try: + if thumb_path.stat().st_mtime >= source.stat().st_mtime: + return thumb_path + except OSError: + pass + + thumb_path.parent.mkdir(parents=True, exist_ok=True) + try: + with Image.open(source) as im: + im = ImageOps.exif_transpose(im) + im.thumbnail((size, size), Image.LANCZOS) + if im.mode not in ("RGB", "L"): + im = im.convert("RGB") + im.save(thumb_path, "JPEG", quality=JPEG_QUALITY, optimize=True) + except (OSError, Image.UnidentifiedImageError): + return None + + return thumb_path diff --git a/templates/documents.html b/templates/documents.html index a35c483..8952f4a 100644 --- a/templates/documents.html +++ b/templates/documents.html @@ -18,6 +18,10 @@
تم رفع الملفات بنجاح وبدأ الاستخراج. قد يستغرق بعض الوقت.
{% endif %} +{% if duplicates_skipped %} +
تم تجاهل {{ duplicates_skipped }} ملف مكرر (موجود مسبقاً في النظام).
+{% endif %} +
{{ stats.total or 0 }}الكل @@ -36,8 +40,24 @@ {{ stats.errors }}أخطاء {% endif %} + {% if stats.duplicates %} + + {{ stats.duplicates }}مكررة + + {% endif %}
+
+ عرض المكررات + +
+ +{% if stats.errors and current_status=='error' %} +
+ +
+{% endif %} + {% if stats.pending_review %}
ابدأ مراجعة {{ stats.pending_review }} وثيقة → @@ -63,7 +83,7 @@ {{ d.id }} - + @@ -72,11 +92,19 @@ {% else %}—{% endif %} {{ d.request_number or '—' }} - {{ d.status }} + + {{ d.status }} + {% if d.duplicate_of %} + مكرر + {% endif %} + {{ d.created_at[:10] if d.created_at else '' }} + {% if d.status == 'error' %} + + {% endif %} - {% if d.status == 'confirmed' %}عرض{% else %}مراجعة{% endif %} + {% if d.status == 'confirmed' %}عرض{% elif d.status == 'error' %}مراجعة{% else %}مراجعة{% endif %} @@ -91,6 +119,23 @@

لا توجد وثائق. ارفع وثائق جديدة

{% endif %} + +{% if total_pages and total_pages > 1 %} +{% set qs = ('status=' ~ current_status ~ '&') if current_status else '' %} + +{% endif %} {% endblock %} {% block scripts %} @@ -108,5 +153,53 @@ async function deleteDoc(docId, btn) { btn.textContent = 'حذف'; } } + +async function retryDoc(docId, btn) { + btn.disabled = true; + btn.textContent = '...'; + const res = await fetch(`/documents/${docId}/retry`, { method: 'POST' }); + if (res.ok) { + location.reload(); + } else { + alert('تعذّرت إعادة المحاولة.'); + btn.disabled = false; + btn.textContent = 'إعادة محاولة'; + } +} + +async function retryAllErrors(btn) { + if (!confirm('إعادة معالجة جميع الوثائق التي بها أخطاء؟')) return; + btn.disabled = true; + const original = btn.textContent; + btn.textContent = 'جارٍ الإرسال...'; + const res = await fetch('/documents/retry-errors', { method: 'POST' }); + if (res.ok) { + location.href = '/documents?status=pending'; + } else { + alert('فشلت إعادة المعالجة.'); + btn.disabled = false; + btn.textContent = original; + } +} + +async function scanDuplicates(btn) { + btn.disabled = true; + const original = btn.textContent; + btn.textContent = 'جارٍ الفحص...'; + const res = await fetch('/documents/scan-duplicates', { method: 'POST' }); + if (res.ok) { + const data = await res.json(); + alert(`اكتمل الفحص:\n- تم احتساب hash لـ ${data.hashed} وثيقة\n- تكرارات (نفس الصورة): ${data.flagged_by_hash}\n- تكرارات (نفس رقم الطلب): ${data.flagged_by_request}\n- الإجمالي المكرر: ${data.total_duplicates}`); + if (data.total_duplicates > 0) { + location.href = '/documents/duplicates'; + } else { + location.reload(); + } + } else { + alert('فشل الفحص.'); + btn.disabled = false; + btn.textContent = original; + } +} {% endblock %} diff --git a/templates/duplicates.html b/templates/duplicates.html new file mode 100644 index 0000000..1b69c55 --- /dev/null +++ b/templates/duplicates.html @@ -0,0 +1,113 @@ +{% extends "base.html" %} +{% block title %}الوثائق المكررة — سجل العقارات{% endblock %} + +{% block content %} +
+
+ مراجعة التكرارات +

الوثائق المكررة

+

وثائق تمّ رصدها كتكرار لوثيقة موجودة مسبقاً. يمكن حذفها بأمان.

+
+
+ + {% if documents %} + + {% endif %} +
+
+ +{% if not documents %} +
+

لا توجد وثائق مكررة حالياً.

+

+
+{% else %} +
+ + + + + + + + + + + + + + {% for d in documents %} + + + + + + + + + + {% endfor %} + +
#الصورةالشخصرقم الطلبمكرر للوثيقةالحالة
{{ d.id }} + + + {% if d.person_id %} + {{ d.first_name or '' }} {{ d.family_name or '' }} + {% else %}—{% endif %} + {{ d.request_number or '—' }} + #{{ d.duplicate_of }} + {% if d.orig_created_at %}
{{ d.orig_created_at[:10] }}{% endif %} + 		{{ d.status }} + عرض + +
+
+{% endif %} +{% endblock %} + +{% block scripts %} + +{% endblock %} diff --git a/templates/person_detail.html b/templates/person_detail.html index 233c918..b959884 100644 --- a/templates/person_detail.html +++ b/templates/person_detail.html @@ -101,7 +101,7 @@
{% for d in documents %} - وثيقة + وثيقة
{% if d.request_number %}طلب {{ d.request_number }}{% endif %} {% if d.request_date %}
{{ d.request_date }}{% endif %}