perf: paginate docs list, lazy thumbnails, static cache headers

database/schema.py

@@ -16,11 +16,16 @@ def _migrate(conn):
         ("registry_office", "TEXT"),
         ("owns_properties", "BOOLEAN"),
         ("declared_property_count", "INTEGER"),
+        ("image_hash", "TEXT"),
+        ("duplicate_of", "INTEGER"),
     ]
     for col, col_type in migrations:
         if col not in existing:
             conn.execute(f"ALTER TABLE documents ADD COLUMN {col} {col_type}")
 
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_documents_image_hash ON documents(image_hash)")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_documents_request_number ON documents(request_number)")
+
     cursor = conn.execute("PRAGMA table_info(properties)")
     existing_prop = {row[1] for row in cursor.fetchall()}
     migrations_prop = [

main.py

@@ -3,11 +3,27 @@ from pathlib import Path
 from fastapi import FastAPI, Request
 from fastapi.responses import RedirectResponse
 from fastapi.staticfiles import StaticFiles
+from starlette.types import Scope
 
 from config import UPLOAD_DIR, ENVIRONMENT
 from database.schema import create_tables
 from routers import documents, review, search, upload, auth
 
+
+class CachedStaticFiles(StaticFiles):
+    """StaticFiles that sets a long Cache-Control header so browsers
+    don't re-download the same image on every refresh."""
+
+    def __init__(self, *args, max_age: int = 86400, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._max_age = max_age
+
+    async def get_response(self, path: str, scope: Scope):
+        response = await super().get_response(path, scope)
+        if response.status_code == 200:
+            response.headers["Cache-Control"] = f"public, max-age={self._max_age}"
+        return response
+
 # Disable docs/openapi in production
 if ENVIRONMENT == "production":
     app = FastAPI(title="Lebanese Real Estate Registry", docs_url=None, redoc_url=None, openapi_url=None)
@@ -39,8 +55,8 @@ async def startup():
     # A script should be used for initial setup.
 
 
-app.mount("/uploads", StaticFiles(directory=UPLOAD_DIR), name="uploads")
-app.mount("/static", StaticFiles(directory="static"), name="static")
+app.mount("/uploads", CachedStaticFiles(directory=UPLOAD_DIR, max_age=604800), name="uploads")
+app.mount("/static", CachedStaticFiles(directory="static", max_age=86400), name="static")
 
 app.include_router(upload.router)
 app.include_router(review.router)

routers/documents.py

@@ -2,35 +2,93 @@ import os
 from pathlib import Path
 
 from fastapi import APIRouter, Request
-from fastapi.responses import JSONResponse
+from fastapi.responses import FileResponse, JSONResponse
 from fastapi.templating import Jinja2Templates
 
 from config import UPLOAD_DIR
 from database.connection import get_db
+from services.thumbnails import get_or_create_thumbnail
 
 router = APIRouter()
 templates = Jinja2Templates(directory="templates")
 
 
+@router.get("/thumbs/{rel_path:path}")
+async def serve_thumbnail(rel_path: str, size: int = 320):
+    """Serve a cached JPEG thumbnail for an uploaded image.
+    Falls back to the original file if thumbnail generation fails."""
+    # Clamp size to sensible values to prevent DoS
+    if size not in (160, 240, 320, 480, 640):
+        size = 320
+
+    thumb = get_or_create_thumbnail(rel_path, size=size)
+    if thumb is None:
+        # Fallback: serve original if it exists
+        base = Path(UPLOAD_DIR).resolve()
+        full = (base / rel_path).resolve()
+        try:
+            full.relative_to(base)
+        except ValueError:
+            return JSONResponse({"error": "not found"}, status_code=404)
+        if not full.is_file():
+            return JSONResponse({"error": "not found"}, status_code=404)
+        return FileResponse(
+            full,
+            headers={"Cache-Control": "public, max-age=604800"},
+        )
+
+    return FileResponse(
+        thumb,
+        media_type="image/jpeg",
+        headers={"Cache-Control": "public, max-age=604800, immutable"},
+    )
+
+_DOC_LIST_COLUMNS = (
+    "d.id, d.person_id, d.image_path, d.request_number, d.status, "
+    "d.duplicate_of, d.extraction_error, d.created_at"
+)
+
+PAGE_SIZE = 50
+
+
 @router.get("/documents")
-async def document_queue(request: Request, status: str = "", uploaded: int = 0):
+async def document_queue(
+    request: Request,
+    status: str = "",
+    uploaded: int = 0,
+    duplicates: int = 0,
+    page: int = 1,
+):
+    if page < 1:
+        page = 1
+    offset = (page - 1) * PAGE_SIZE
+
     with get_db() as conn:
         if status:
             rows = conn.execute(
-                """SELECT d.*, p.first_name, p.family_name
+                f"""SELECT {_DOC_LIST_COLUMNS}, p.first_name, p.family_name
                    FROM documents d
                    LEFT JOIN persons p ON p.id = d.person_id
                    WHERE d.status=?
-                   ORDER BY d.id DESC""",
-                (status,),
+                   ORDER BY d.id DESC
+                   LIMIT ? OFFSET ?""",
+                (status, PAGE_SIZE, offset),
             ).fetchall()
+            total_filtered = conn.execute(
+                "SELECT COUNT(*) AS n FROM documents WHERE status=?", (status,)
+            ).fetchone()["n"]
         else:
             rows = conn.execute(
-                """SELECT d.*, p.first_name, p.family_name
+                f"""SELECT {_DOC_LIST_COLUMNS}, p.first_name, p.family_name
                    FROM documents d
                    LEFT JOIN persons p ON p.id = d.person_id
-                   ORDER BY d.id DESC"""
+                   ORDER BY d.id DESC
+                   LIMIT ? OFFSET ?""",
+                (PAGE_SIZE, offset),
             ).fetchall()
+            total_filtered = conn.execute(
+                "SELECT COUNT(*) AS n FROM documents"
+            ).fetchone()["n"]
 
         stats = conn.execute(
             """SELECT
@@ -38,10 +96,13 @@ async def document_queue(request: Request, status: str = "", uploaded: int = 0):
                SUM(CASE WHEN status='confirmed' THEN 1 ELSE 0 END) AS confirmed,
                SUM(CASE WHEN status='extracted' THEN 1 ELSE 0 END) AS pending_review,
                SUM(CASE WHEN status='pending' THEN 1 ELSE 0 END) AS processing,
-               SUM(CASE WHEN status='error' THEN 1 ELSE 0 END) AS errors
+               SUM(CASE WHEN status='error' THEN 1 ELSE 0 END) AS errors,
+               SUM(CASE WHEN duplicate_of IS NOT NULL THEN 1 ELSE 0 END) AS duplicates
             FROM documents WHERE status != 'staged'"""
         ).fetchone()
 
+    total_pages = max(1, (total_filtered + PAGE_SIZE - 1) // PAGE_SIZE)
+
     return templates.TemplateResponse(
         request,
         "documents.html",
@@ -50,6 +111,11 @@ async def document_queue(request: Request, status: str = "", uploaded: int = 0):
             "stats": dict(stats) if stats else {},
             "current_status": status,
             "uploaded": uploaded,
+            "duplicates_skipped": duplicates,
+            "page": page,
+            "total_pages": total_pages,
+            "page_size": PAGE_SIZE,
+            "total_filtered": total_filtered,
         },
     )
 
@@ -88,3 +154,196 @@ async def delete_document(doc_id: int):
         pass
 
     return JSONResponse({"ok": True})
+
+
+@router.post("/documents/{doc_id}/retry")
+async def retry_document(doc_id: int):
+    """Re-run AI extraction on an errored or extracted document."""
+    import asyncio
+    from services.extractor import get_default_provider
+    from routers.upload import _extract_and_save
+
+    with get_db() as conn:
+        doc = conn.execute(
+            "SELECT id, image_path, provider, status FROM documents WHERE id=?",
+            (doc_id,),
+        ).fetchone()
+        if not doc:
+            return JSONResponse({"error": "not found"}, status_code=404)
+
+        # Clear previous extraction data and requeue
+        conn.execute("DELETE FROM properties WHERE document_id=?", (doc_id,))
+        conn.execute(
+            """UPDATE documents SET status='pending',
+                   extraction_error=NULL,
+                   raw_extraction_json=NULL,
+                   updated_at=CURRENT_TIMESTAMP
+               WHERE id=?""",
+            (doc_id,),
+        )
+
+    provider = doc["provider"] or get_default_provider()
+    asyncio.create_task(_extract_and_save(doc_id, doc["image_path"], provider))
+    return JSONResponse({"ok": True, "status": "pending"})
+
+
+@router.post("/documents/retry-errors")
+async def retry_all_errors():
+    """Re-run extraction for every document currently in error state."""
+    import asyncio
+    from services.extractor import get_default_provider
+    from routers.upload import _extract_and_save
+
+    default_provider = get_default_provider()
+    with get_db() as conn:
+        rows = conn.execute(
+            "SELECT id, image_path, provider FROM documents WHERE status='error'"
+        ).fetchall()
+        ids = [r["id"] for r in rows]
+        if ids:
+            conn.execute("DELETE FROM properties WHERE document_id IN (" + ",".join("?" * len(ids)) + ")", ids)
+            conn.execute(
+                "UPDATE documents SET status='pending', extraction_error=NULL, raw_extraction_json=NULL, updated_at=CURRENT_TIMESTAMP WHERE id IN (" + ",".join("?" * len(ids)) + ")",
+                ids,
+            )
+
+    for row in rows:
+        provider = row["provider"] or default_provider
+        asyncio.create_task(_extract_and_save(row["id"], row["image_path"], provider))
+
+    return JSONResponse({"ok": True, "retried": len(rows)})
+
+
+@router.post("/documents/scan-duplicates")
+async def scan_duplicates():
+    """
+    Backfill image_hash for existing documents and flag duplicates.
+    A doc is marked duplicate_of the earliest document (lowest id) that shares
+    either the exact image hash OR the same request_number + search_scope + page_info.
+    """
+    import hashlib
+
+    def _hash_file(path: Path) -> str | None:
+        try:
+            h = hashlib.sha256()
+            with open(path, "rb") as f:
+                for chunk in iter(lambda: f.read(1 << 20), b""):
+                    h.update(chunk)
+            return h.hexdigest()
+        except OSError:
+            return None
+
+    hashed = 0
+    flagged_by_hash = 0
+    flagged_by_request = 0
+
+    with get_db() as conn:
+        # 1. Hash any document missing image_hash
+        rows = conn.execute(
+            "SELECT id, image_path FROM documents WHERE image_hash IS NULL OR image_hash=''"
+        ).fetchall()
+        for row in rows:
+            full_path = Path(UPLOAD_DIR) / row["image_path"]
+            digest = _hash_file(full_path)
+            if digest:
+                conn.execute(
+                    "UPDATE documents SET image_hash=? WHERE id=?",
+                    (digest, row["id"]),
+                )
+                hashed += 1
+
+        # 2. Flag duplicates by image_hash (keep earliest)
+        hash_groups = conn.execute(
+            """SELECT image_hash, MIN(id) AS keeper, COUNT(*) AS n
+               FROM documents
+               WHERE image_hash IS NOT NULL AND image_hash != ''
+                 AND status != 'staged'
+               GROUP BY image_hash
+               HAVING n > 1"""
+        ).fetchall()
+        for g in hash_groups:
+            result = conn.execute(
+                """UPDATE documents SET duplicate_of=?
+                   WHERE image_hash=? AND id != ? AND status != 'staged'""",
+                (g["keeper"], g["image_hash"], g["keeper"]),
+            )
+            flagged_by_hash += result.rowcount or 0
+
+        # 3. Flag duplicates by request_number + search_scope + page_info (keep earliest)
+        logical_groups = conn.execute(
+            """SELECT TRIM(request_number) AS rn,
+                      COALESCE(TRIM(search_scope),'') AS sc,
+                      COALESCE(TRIM(page_info),'') AS pi,
+                      MIN(id) AS keeper,
+                      COUNT(*) AS n
+               FROM documents
+               WHERE status IN ('extracted','confirmed')
+                 AND request_number IS NOT NULL AND TRIM(request_number) != ''
+               GROUP BY rn, sc, pi
+               HAVING n > 1"""
+        ).fetchall()
+        for g in logical_groups:
+            result = conn.execute(
+                """UPDATE documents SET duplicate_of=?
+                   WHERE TRIM(request_number)=?
+                     AND COALESCE(TRIM(search_scope),'')=?
+                     AND COALESCE(TRIM(page_info),'')=?
+                     AND id != ?
+                     AND duplicate_of IS NULL
+                     AND status IN ('extracted','confirmed')""",
+                (g["keeper"], g["rn"], g["sc"], g["pi"], g["keeper"]),
+            )
+            flagged_by_request += result.rowcount or 0
+
+        total_dupes = conn.execute(
+            "SELECT COUNT(*) AS n FROM documents WHERE duplicate_of IS NOT NULL"
+        ).fetchone()["n"]
+
+    return JSONResponse({
+        "ok": True,
+        "hashed": hashed,
+        "flagged_by_hash": flagged_by_hash,
+        "flagged_by_request": flagged_by_request,
+        "total_duplicates": total_dupes,
+    })
+
+
+@router.get("/documents/duplicates")
+async def duplicates_view(request: Request):
+    """List all documents flagged as duplicates alongside their originals."""
+    with get_db() as conn:
+        rows = conn.execute(
+            """SELECT d.*, p.first_name, p.family_name,
+                      o.request_number AS orig_request_number,
+                      o.created_at AS orig_created_at
+               FROM documents d
+               LEFT JOIN persons p ON p.id = d.person_id
+               LEFT JOIN documents o ON o.id = d.duplicate_of
+               WHERE d.duplicate_of IS NOT NULL
+               ORDER BY d.duplicate_of, d.id"""
+        ).fetchall()
+    return templates.TemplateResponse(
+        request,
+        "duplicates.html",
+        {"documents": [dict(r) for r in rows]},
+    )
+
+
+@router.post("/documents/delete-duplicates")
+async def delete_all_duplicates():
+    """Delete every document flagged as duplicate_of another doc."""
+    deleted = 0
+    with get_db() as conn:
+        rows = conn.execute(
+            "SELECT id, image_path FROM documents WHERE duplicate_of IS NOT NULL"
+        ).fetchall()
+        for row in rows:
+            conn.execute("DELETE FROM properties WHERE document_id=?", (row["id"],))
+            conn.execute("DELETE FROM documents WHERE id=?", (row["id"],))
+            # Remove file from disk
+            try:
+                (Path(UPLOAD_DIR) / row["image_path"]).unlink(missing_ok=True)
+            except OSError:
+                pass
+            deleted += 1
+    return JSONResponse({"ok": True, "deleted": deleted})

routers/upload.py

@@ -1,11 +1,12 @@
 import asyncio
+import hashlib
 import json
 import uuid
 from datetime import date
 from pathlib import Path
 
 from fastapi import APIRouter, File, Form, Request, UploadFile
-from fastapi.responses import RedirectResponse
+from fastapi.responses import JSONResponse, RedirectResponse
 from fastapi.templating import Jinja2Templates
 
 from config import UPLOAD_DIR
@@ -21,6 +22,30 @@ ALLOWED_PDF_EXTS = {".pdf"}
 ALLOWED_EXTENSIONS = ALLOWED_IMAGE_EXTS | ALLOWED_PDF_EXTS
 
 
+def _hash_bytes(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def _hash_file(path: Path) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def _find_duplicate(conn, image_hash: str) -> dict | None:
+    """Return an existing non-staged document sharing the same image hash."""
+    row = conn.execute(
+        """SELECT id, status, image_path, person_id, request_number
+           FROM documents
+           WHERE image_hash=? AND status != 'staged'
+           ORDER BY id LIMIT 1""",
+        (image_hash,),
+    ).fetchone()
+    return dict(row) if row else None
+
+
 def _save_image(file_bytes: bytes, original_name: str) -> str:
     """Save image to uploads/{date}/{uuid}_{name} and return relative path."""
     today = date.today().isoformat()
@@ -90,6 +115,26 @@ async def _extract_and_save(doc_id: int, image_path: str, provider: str = ""):
                     ),
                 )
 
+            # Flag logical duplicates: same request_number + search_scope + page_number
+            req_num = (data.get("request_number") or "").strip()
+            scope = (data.get("search_scope") or "").strip()
+            page_info = (data.get("page_info") or "").strip()
+            if req_num:
+                existing = conn.execute(
+                    """SELECT id FROM documents
+                       WHERE id != ? AND request_number=?
+                         AND COALESCE(search_scope,'')=?
+                         AND COALESCE(page_info,'')=?
+                         AND status IN ('extracted','confirmed')
+                       ORDER BY id LIMIT 1""",
+                    (doc_id, req_num, scope, page_info),
+                ).fetchone()
+                if existing:
+                    conn.execute(
+                        "UPDATE documents SET duplicate_of=? WHERE id=?",
+                        (existing["id"], doc_id),
+                    )
+
     except Exception as e:
         with get_db() as conn:
             conn.execute(
@@ -129,6 +174,7 @@ async def upload_files(
         provider = get_default_provider()
 
     doc_ids = []
+    duplicates = []
 
     for upload in files:
         suffix = Path(upload.filename).suffix.lower()
@@ -138,16 +184,32 @@ async def upload_files(
         file_bytes = await upload.read()
 
         if suffix in ALLOWED_PDF_EXTS:
-            # PDF: split into per-page images
+            # PDF: split into per-page images; hash each rendered page
             pages = pdf_to_images(file_bytes, upload.filename)
             for page_info in pages:
+                page_path = Path(UPLOAD_DIR) / page_info["image_path"]
+                page_hash = _hash_file(page_path)
                 with get_db() as conn:
+                    dup = _find_duplicate(conn, page_hash)
+                    if dup:
+                        # Discard the freshly rendered duplicate page
+                        try:
+                            page_path.unlink(missing_ok=True)
+                        except OSError:
+                            pass
+                        duplicates.append({
+                            "name": f"{upload.filename} (p{page_info['page_number']})",
+                            "existing_id": dup["id"],
+                            "status": dup["status"],
+                        })
+                        continue
                     cursor = conn.execute(
                         """INSERT INTO documents
-                           (image_path, status, provider, pdf_group_id, page_number)
-                           VALUES (?, 'pending', ?, ?, ?)""",
+                           (image_path, image_hash, status, provider, pdf_group_id, page_number)
+                           VALUES (?, ?, 'pending', ?, ?, ?)""",
                         (
                             page_info["image_path"],
+                            page_hash,
                             provider,
                             page_info["pdf_group_id"],
                             page_info["page_number"],
@@ -156,11 +218,20 @@ async def upload_files(
                     doc_ids.append((cursor.lastrowid, page_info["image_path"]))
         else:
             # Image file
-            rel_path = _save_image(file_bytes, upload.filename)
+            image_hash = _hash_bytes(file_bytes)
             with get_db() as conn:
+                dup = _find_duplicate(conn, image_hash)
+                if dup:
+                    duplicates.append({
+                        "name": upload.filename,
+                        "existing_id": dup["id"],
+                        "status": dup["status"],
+                    })
+                    continue
+                rel_path = _save_image(file_bytes, upload.filename)
                 cursor = conn.execute(
-                    "INSERT INTO documents (image_path, status, provider) VALUES (?, 'pending', ?)",
-                    (rel_path, provider),
+                    "INSERT INTO documents (image_path, image_hash, status, provider) VALUES (?, ?, 'pending', ?)",
+                    (rel_path, image_hash, provider),
                 )
                 doc_ids.append((cursor.lastrowid, rel_path))
 
@@ -168,9 +239,16 @@ async def upload_files(
     for doc_id, rel_path in doc_ids:
         asyncio.create_task(_extract_and_save(doc_id, rel_path, provider))
 
-    if len(doc_ids) == 1:
+    if len(doc_ids) == 1 and not duplicates:
         return RedirectResponse(f"/review/{doc_ids[0][0]}?wait=1", status_code=303)
-    return RedirectResponse("/documents?uploaded=1", status_code=303)
+
+    query = "uploaded=1"
+    if duplicates:
+        query += f"&duplicates={len(duplicates)}"
+        # Route user to the first duplicate so they can see which doc matched
+        if not doc_ids:
+            return RedirectResponse(f"/review/{duplicates[0]['existing_id']}", status_code=303)
+    return RedirectResponse(f"/documents?{query}", status_code=303)
 
 
 # ─── Two-step workflow: stage images, then process ─────────────
@@ -182,6 +260,7 @@ async def stage_file(
 ):
     """Save uploaded images without triggering AI extraction."""
     staged = []
+    duplicates = []
     for upload in files:
         suffix = Path(upload.filename).suffix.lower()
         if suffix not in ALLOWED_EXTENSIONS:
@@ -192,13 +271,28 @@ async def stage_file(
         if suffix in ALLOWED_PDF_EXTS:
             pages = pdf_to_images(file_bytes, upload.filename)
             for page_info in pages:
+                page_path = Path(UPLOAD_DIR) / page_info["image_path"]
+                page_hash = _hash_file(page_path)
                 with get_db() as conn:
+                    dup = _find_duplicate(conn, page_hash)
+                    if dup:
+                        try:
+                            page_path.unlink(missing_ok=True)
+                        except OSError:
+                            pass
+                        duplicates.append({
+                            "name": f"{upload.filename} (p{page_info['page_number']})",
+                            "existing_id": dup["id"],
+                            "status": dup["status"],
+                        })
+                        continue
                     cursor = conn.execute(
                         """INSERT INTO documents
-                           (image_path, status, pdf_group_id, page_number)
-                           VALUES (?, 'staged', ?, ?)""",
+                           (image_path, image_hash, status, pdf_group_id, page_number)
+                           VALUES (?, ?, 'staged', ?, ?)""",
                         (
                             page_info["image_path"],
+                            page_hash,
                             page_info["pdf_group_id"],
                             page_info["page_number"],
                         ),
@@ -209,11 +303,20 @@ async def stage_file(
                         "name": f"{upload.filename} (p{page_info['page_number']})",
                     })
         else:
-            rel_path = _save_image(file_bytes, upload.filename)
+            image_hash = _hash_bytes(file_bytes)
             with get_db() as conn:
+                dup = _find_duplicate(conn, image_hash)
+                if dup:
+                    duplicates.append({
+                        "name": upload.filename,
+                        "existing_id": dup["id"],
+                        "status": dup["status"],
+                    })
+                    continue
+                rel_path = _save_image(file_bytes, upload.filename)
                 cursor = conn.execute(
-                    "INSERT INTO documents (image_path, status) VALUES (?, 'staged')",
-                    (rel_path,),
+                    "INSERT INTO documents (image_path, image_hash, status) VALUES (?, ?, 'staged')",
+                    (rel_path, image_hash),
                 )
                 staged.append({
                     "id": cursor.lastrowid,
@@ -221,8 +324,7 @@ async def stage_file(
                     "name": upload.filename,
                 })
 
-    from fastapi.responses import JSONResponse
-    return JSONResponse({"staged": staged})
+    return JSONResponse({"staged": staged, "duplicates": duplicates})
 
 
 @router.post("/upload/process-staged")
@@ -259,7 +361,6 @@ async def process_staged(
 @router.delete("/upload/staged/{doc_id}")
 async def remove_staged(doc_id: int):
     """Remove a single staged document before processing."""
-    from fastapi.responses import JSONResponse
     with get_db() as conn:
         row = conn.execute("SELECT image_path FROM documents WHERE id=? AND status='staged'", (doc_id,)).fetchone()
         if row:

services/extractor.py

@@ -131,8 +131,7 @@ async def verify_page_correlation(
                 }
             ],
         )
-        raw_text = _strip_code_fences(message.content[0].text)
-        result = json.loads(raw_text)
+        result = _parse_json_lenient(message.content[0].text)
     elif provider == "gemini":
         import asyncio
         from google import genai
@@ -176,8 +175,7 @@ async def verify_page_correlation(
             return response.text
 
         raw_text = await asyncio.get_event_loop().run_in_executor(None, _call)
-        raw_text = _strip_code_fences(raw_text)
-        result = json.loads(raw_text)
+        result = _parse_json_lenient(raw_text)
     else:
         raise ValueError(f"Provider {provider} does not support AI correlation verification")
 
@@ -221,6 +219,36 @@ def _strip_code_fences(text: str) -> str:
     return text.strip()
 
 
+def _parse_json_lenient(text: str) -> dict:
+    """Parse JSON even if the model added prose, trailing commas, or truncation."""
+    text = _strip_code_fences(text or "")
+    if not text:
+        raise ValueError("Empty response from model")
+
+    # Fast path
+    try:
+        return json.loads(text)
+    except Exception:
+        pass
+
+    # Extract the first {...} block (greedy to last })
+    start = text.find("{")
+    end = text.rfind("}")
+    if start != -1 and end > start:
+        candidate = text[start:end + 1]
+        try:
+            return json.loads(candidate)
+        except Exception:
+            # Remove trailing commas before } or ]
+            cleaned = re.sub(r",(\s*[}\]])", r"\1", candidate)
+            try:
+                return json.loads(cleaned)
+            except Exception:
+                pass
+
+    raise ValueError(f"Could not parse model response as JSON: {text[:200]}")
+
+
 def get_available_providers() -> list[dict]:
     """Return list of all available providers (EasyOCR is always available)."""
     providers = [
@@ -248,41 +276,55 @@ def get_default_provider() -> str:
 
 async def _extract_with_claude(image_path: str) -> dict:
     import anthropic
+    import asyncio
 
     client = anthropic.AsyncAnthropic(api_key=ANTHROPIC_API_KEY)
     full_path = _resolve_path(image_path)
     img_data, media_type = _encode_image(full_path)
 
-    message = await client.messages.create(
-        model=CLAUDE_MODEL,
-        max_tokens=4096,
-        system=[
-            {
-                "type": "text",
-                "text": SYSTEM_PROMPT,
-                "cache_control": {"type": "ephemeral"},
-            }
-        ],
-        messages=[
-            {
-                "role": "user",
-                "content": [
+    last_error = None
+    for attempt in range(3):
+        try:
+            message = await client.messages.create(
+                model=CLAUDE_MODEL,
+                max_tokens=8192,
+                system=[
                     {
-                        "type": "image",
-                        "source": {
-                            "type": "base64",
-                            "media_type": media_type,
-                            "data": img_data,
-                        },
-                    },
-                    {"type": "text", "text": USER_PROMPT},
+                        "type": "text",
+                        "text": SYSTEM_PROMPT,
+                        "cache_control": {"type": "ephemeral"},
+                    }
                 ],
-            }
-        ],
-    )
-
-    raw_text = _strip_code_fences(message.content[0].text)
-    return json.loads(raw_text)
+                messages=[
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "image",
+                                "source": {
+                                    "type": "base64",
+                                    "media_type": media_type,
+                                    "data": img_data,
+                                },
+                            },
+                            {"type": "text", "text": USER_PROMPT},
+                        ],
+                    }
+                ],
+            )
+            return _parse_json_lenient(message.content[0].text)
+        except Exception as e:
+            last_error = e
+            err_str = str(e)
+            # Transient: overload/rate-limit/network — back off and retry
+            if any(code in err_str for code in ("529", "503", "502", "500", "overload", "Overload", "timeout", "Timeout")):
+                await asyncio.sleep(2 * (attempt + 1))
+                continue
+            if "429" in err_str and attempt < 2:
+                await asyncio.sleep(5 * (attempt + 1))
+                continue
+            raise
+    raise last_error
 
 
 # ─── Gemini extraction ───────────────────────────────────────────
@@ -320,24 +362,36 @@ async def _extract_with_gemini(image_path: str) -> dict:
                 max_output_tokens=8192,
             ),
         )
-        return response.text
+        text = response.text
+        if not text:
+            try:
+                parts = []
+                for cand in (response.candidates or []):
+                    for part in (cand.content.parts or []):
+                        if getattr(part, "text", None):
+                            parts.append(part.text)
+                text = "".join(parts)
+            except Exception:
+                pass
+        if not text:
+            raise ValueError("Gemini returned empty response")
+        return text
 
     last_error = None
     for model_name in models_to_try:
-        for attempt in range(2):
+        for attempt in range(3):
             try:
                 raw_text = await asyncio.get_event_loop().run_in_executor(None, lambda m=model_name: _call(m))
-                raw_text = _strip_code_fences(raw_text)
-                return json.loads(raw_text)
+                return _parse_json_lenient(raw_text)
             except Exception as e:
                 last_error = e
                 err_str = str(e)
-                # On quota exhaustion, skip to next model immediately (don't waste retries)
+                # Quota exhaustion — skip to next model immediately
                 if "429" in err_str or "RESOURCE_EXHAUSTED" in err_str:
                     break
-                # On temporary overload, wait and retry same model
-                if "503" in err_str or "UNAVAILABLE" in err_str:
-                    await asyncio.sleep(3 * (attempt + 1))
+                # Transient errors — back off and retry same model
+                if any(code in err_str for code in ("503", "502", "500", "UNAVAILABLE", "DEADLINE", "timeout", "Timeout", "empty response", "Could not parse")):
+                    await asyncio.sleep(2 * (attempt + 1))
                     continue
                 raise
 

services/thumbnails.py

@@ -0,0 +1,61 @@
+"""On-demand thumbnail generation with disk caching.
+
+Thumbnails are stored under UPLOAD_DIR/.thumbs/<size>/<original relative path>.jpg
+and generated lazily the first time they are requested. This keeps the
+documents list fast even with hundreds of full-resolution uploads.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+from PIL import Image, ImageOps
+
+from config import UPLOAD_DIR
+
+THUMB_ROOT = Path(UPLOAD_DIR) / ".thumbs"
+DEFAULT_SIZE = 320  # px on the longest edge
+JPEG_QUALITY = 78
+
+
+def _safe_rel(rel_path: str) -> Optional[Path]:
+    """Resolve rel_path against UPLOAD_DIR, rejecting traversal."""
+    base = Path(UPLOAD_DIR).resolve()
+    target = (base / rel_path).resolve()
+    try:
+        target.relative_to(base)
+    except ValueError:
+        return None
+    return target
+
+
+def get_or_create_thumbnail(rel_path: str, size: int = DEFAULT_SIZE) -> Optional[Path]:
+    """Return a path to a cached JPEG thumbnail for the given upload,
+    creating it if needed. Returns None if the source doesn't exist."""
+    source = _safe_rel(rel_path)
+    if source is None or not source.is_file():
+        return None
+
+    thumb_dir = THUMB_ROOT / str(size)
+    thumb_path = thumb_dir / (rel_path + ".jpg")
+
+    # Regenerate if missing or source is newer
+    if thumb_path.is_file():
+        try:
+            if thumb_path.stat().st_mtime >= source.stat().st_mtime:
+                return thumb_path
+        except OSError:
+            pass
+
+    thumb_path.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        with Image.open(source) as im:
+            im = ImageOps.exif_transpose(im)
+            im.thumbnail((size, size), Image.LANCZOS)
+            if im.mode not in ("RGB", "L"):
+                im = im.convert("RGB")
+            im.save(thumb_path, "JPEG", quality=JPEG_QUALITY, optimize=True)
+    except (OSError, Image.UnidentifiedImageError):
+        return None
+
+    return thumb_path

templates/documents.html

@@ -18,6 +18,10 @@
 <div class="success-banner">تم رفع الملفات بنجاح وبدأ الاستخراج. قد يستغرق بعض الوقت.</div>
 {% endif %}
 
+{% if duplicates_skipped %}
+<div class="alert-banner">تم تجاهل {{ duplicates_skipped }} ملف مكرر (موجود مسبقاً في النظام).</div>
+{% endif %}
+
 <div class="stats-bar">
   <a href="/documents" class="stat {{ 'active' if not current_status }}">
     <span class="stat-num">{{ stats.total or 0 }}</span><span class="stat-label">الكل</span>
@@ -36,8 +40,24 @@
     <span class="stat-num">{{ stats.errors }}</span><span class="stat-label">أخطاء</span>
   </a>
   {% endif %}
+  {% if stats.duplicates %}
+  <a href="/documents/duplicates" class="stat">
+    <span class="stat-num">{{ stats.duplicates }}</span><span class="stat-label">مكررة</span>
+  </a>
+  {% endif %}
 </div>
 
+<div style="display:flex;gap:.5rem;margin:.75rem 0;flex-wrap:wrap">
+  <a href="/documents/duplicates" class="btn btn-sm btn-secondary">عرض المكررات</a>
+  <button class="btn btn-sm btn-secondary" onclick="scanDuplicates(this)">فحص التكرارات في قاعدة البيانات</button>
+</div>
+
+{% if stats.errors and current_status=='error' %}
+<div class="alert-banner">
+  <button class="btn btn-sm btn-primary" onclick="retryAllErrors(this)">إعادة معالجة جميع الأخطاء ({{ stats.errors }})</button>
+</div>
+{% endif %}
+
 {% if stats.pending_review %}
 <div class="alert-banner">
   <a href="/review/next">ابدأ مراجعة {{ stats.pending_review }} وثيقة →</a>
@@ -63,7 +83,7 @@
         <td data-label="#">{{ d.id }}</td>
         <td data-label="الصورة">
           <a href="/review/{{ d.id }}" title="فتح الوثيقة {{ d.id }} للمراجعة" aria-label="فتح الوثيقة {{ d.id }} للمراجعة">
-            <img src="/uploads/{{ d.image_path }}" class="doc-thumb-mini" alt="">
+            <img src="/thumbs/{{ d.image_path }}" class="doc-thumb-mini" alt="" loading="lazy" decoding="async" width="80" height="80">
           </a>
         </td>
         <td data-label="الشخص">
@@ -72,11 +92,19 @@
           {% else %}—{% endif %}
         </td>
         <td data-label="رقم الطلب">{{ d.request_number or '—' }}</td>
-        <td data-label="الحالة"><span class="status-badge status-{{ d.status }}">{{ d.status }}</span></td>
+        <td data-label="الحالة">
+          <span class="status-badge status-{{ d.status }}">{{ d.status }}</span>
+          {% if d.duplicate_of %}
+            <a href="/review/{{ d.duplicate_of }}" class="status-badge" style="background:#fde68a;color:#92400e;margin-inline-start:.25rem" title="مكرر للوثيقة رقم {{ d.duplicate_of }}">مكرر</a>
+          {% endif %}
+        </td>
         <td data-label="التاريخ">{{ d.created_at[:10] if d.created_at else '' }}</td>
         <td data-label="الإجراءات" class="doc-actions">
+          {% if d.status == 'error' %}
+          <button class="btn btn-sm btn-primary" onclick="retryDoc({{ d.id }}, this)" title="{{ d.extraction_error or '' }}">إعادة محاولة</button>
+          {% endif %}
           <a href="/review/{{ d.id }}" class="btn btn-sm btn-secondary">
-            {% if d.status == 'confirmed' %}عرض{% else %}مراجعة{% endif %}
+            {% if d.status == 'confirmed' %}عرض{% elif d.status == 'error' %}مراجعة{% else %}مراجعة{% endif %}
           </a>
           <button class="btn btn-sm btn-danger" onclick="deleteDoc({{ d.id }}, this)">حذف</button>
         </td>
@@ -91,6 +119,23 @@
   <p>لا توجد وثائق. <a href="/">ارفع وثائق جديدة</a></p>
 </div>
 {% endif %}
+
+{% if total_pages and total_pages > 1 %}
+{% set qs = ('status=' ~ current_status ~ '&') if current_status else '' %}
+<div class="pagination" style="display:flex;gap:.5rem;justify-content:center;align-items:center;margin:1rem 0;flex-wrap:wrap">
+  {% if page > 1 %}
+    <a class="btn btn-sm btn-secondary" href="?{{ qs }}page={{ page - 1 }}">السابق</a>
+  {% else %}
+    <span class="btn btn-sm btn-secondary" style="opacity:.5;pointer-events:none">السابق</span>
+  {% endif %}
+  <span style="font-size:.9rem">صفحة {{ page }} من {{ total_pages }} — {{ total_filtered }} وثيقة</span>
+  {% if page < total_pages %}
+    <a class="btn btn-sm btn-secondary" href="?{{ qs }}page={{ page + 1 }}">التالي</a>
+  {% else %}
+    <span class="btn btn-sm btn-secondary" style="opacity:.5;pointer-events:none">التالي</span>
+  {% endif %}
+</div>
+{% endif %}
 {% endblock %}
 
 {% block scripts %}
@@ -108,5 +153,53 @@ async function deleteDoc(docId, btn) {
     btn.textContent = 'حذف';
   }
 }
+
+async function retryDoc(docId, btn) {
+  btn.disabled = true;
+  btn.textContent = '...';
+  const res = await fetch(`/documents/${docId}/retry`, { method: 'POST' });
+  if (res.ok) {
+    location.reload();
+  } else {
+    alert('تعذّرت إعادة المحاولة.');
+    btn.disabled = false;
+    btn.textContent = 'إعادة محاولة';
+  }
+}
+
+async function retryAllErrors(btn) {
+  if (!confirm('إعادة معالجة جميع الوثائق التي بها أخطاء؟')) return;
+  btn.disabled = true;
+  const original = btn.textContent;
+  btn.textContent = 'جارٍ الإرسال...';
+  const res = await fetch('/documents/retry-errors', { method: 'POST' });
+  if (res.ok) {
+    location.href = '/documents?status=pending';
+  } else {
+    alert('فشلت إعادة المعالجة.');
+    btn.disabled = false;
+    btn.textContent = original;
+  }
+}
+
+async function scanDuplicates(btn) {
+  btn.disabled = true;
+  const original = btn.textContent;
+  btn.textContent = 'جارٍ الفحص...';
+  const res = await fetch('/documents/scan-duplicates', { method: 'POST' });
+  if (res.ok) {
+    const data = await res.json();
+    alert(`اكتمل الفحص:\n- تم احتساب hash لـ ${data.hashed} وثيقة\n- تكرارات (نفس الصورة): ${data.flagged_by_hash}\n- تكرارات (نفس رقم الطلب): ${data.flagged_by_request}\n- الإجمالي المكرر: ${data.total_duplicates}`);
+    if (data.total_duplicates > 0) {
+      location.href = '/documents/duplicates';
+    } else {
+      location.reload();
+    }
+  } else {
+    alert('فشل الفحص.');
+    btn.disabled = false;
+    btn.textContent = original;
+  }
+}
 </script>
 {% endblock %}

templates/duplicates.html

@@ -0,0 +1,113 @@
+{% extends "base.html" %}
+{% block title %}الوثائق المكررة — سجل العقارات{% endblock %}
+
+{% block content %}
+<div class="page-header">
+  <div>
+    <span class="page-kicker">مراجعة التكرارات</span>
+    <h1>الوثائق المكررة</h1>
+    <p class="subtitle">وثائق تمّ رصدها كتكرار لوثيقة موجودة مسبقاً. يمكن حذفها بأمان.</p>
+  </div>
+  <div style="display:flex;gap:.5rem">
+    <button class="btn btn-secondary" onclick="scanDuplicates(this)">فحص التكرارات الآن</button>
+    {% if documents %}
+    <button class="btn btn-danger" onclick="deleteAllDuplicates(this)">حذف جميع التكرارات ({{ documents|length }})</button>
+    {% endif %}
+  </div>
+</div>
+
+{% if not documents %}
+<div class="no-results">
+  <p>لا توجد وثائق مكررة حالياً.</p>
+  <p><button class="btn btn-primary" onclick="scanDuplicates(this)">فحص قاعدة البيانات عن التكرارات</button></p>
+</div>
+{% else %}
+<div class="table-wrapper">
+  <table class="results-table responsive-table">
+    <thead>
+      <tr>
+        <th>#</th>
+        <th>الصورة</th>
+        <th>الشخص</th>
+        <th>رقم الطلب</th>
+        <th>مكرر للوثيقة</th>
+        <th>الحالة</th>
+        <th></th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for d in documents %}
+      <tr id="row-{{ d.id }}">
+        <td data-label="#">{{ d.id }}</td>
+        <td data-label="الصورة">
+          <a href="/review/{{ d.id }}"><img src="/thumbs/{{ d.image_path }}" class="doc-thumb-mini" alt="" loading="lazy" decoding="async" width="80" height="80"></a>
+        </td>
+        <td data-label="الشخص">
+          {% if d.person_id %}
+          <a href="/persons/{{ d.person_id }}">{{ d.first_name or '' }} {{ d.family_name or '' }}</a>
+          {% else %}—{% endif %}
+        </td>
+        <td data-label="رقم الطلب">{{ d.request_number or '—' }}</td>
+        <td data-label="مكرر للوثيقة">
+          <a href="/review/{{ d.duplicate_of }}">#{{ d.duplicate_of }}</a>
+          {% if d.orig_created_at %}<br><small>{{ d.orig_created_at[:10] }}</small>{% endif %}
+        </td>
+        <td data-label="الحالة"><span class="status-badge status-{{ d.status }}">{{ d.status }}</span></td>
+        <td data-label="الإجراءات" class="doc-actions">
+          <a href="/review/{{ d.id }}" class="btn btn-sm btn-secondary">عرض</a>
+          <button class="btn btn-sm btn-danger" onclick="deleteDoc({{ d.id }}, this)">حذف</button>
+        </td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+</div>
+{% endif %}
+{% endblock %}
+
+{% block scripts %}
+<script>
+async function scanDuplicates(btn) {
+  btn.disabled = true;
+  const original = btn.textContent;
+  btn.textContent = 'جارٍ الفحص...';
+  const res = await fetch('/documents/scan-duplicates', { method: 'POST' });
+  if (res.ok) {
+    const data = await res.json();
+    alert(`اكتمل الفحص:\n- تم احتساب hash لـ ${data.hashed} وثيقة\n- تكرارات (نفس الصورة): ${data.flagged_by_hash}\n- تكرارات (نفس رقم الطلب): ${data.flagged_by_request}\n- الإجمالي المكرر: ${data.total_duplicates}`);
+    location.reload();
+  } else {
+    alert('فشل الفحص.');
+    btn.disabled = false;
+    btn.textContent = original;
+  }
+}
+
+async function deleteDoc(docId, btn) {
+  if (!confirm('حذف هذه الوثيقة المكررة؟')) return;
+  btn.disabled = true;
+  const res = await fetch(`/documents/${docId}`, { method: 'DELETE' });
+  if (res.ok) {
+    document.getElementById(`row-${docId}`).remove();
+  } else {
+    alert('فشل الحذف.');
+    btn.disabled = false;
+  }
+}
+
+async function deleteAllDuplicates(btn) {
+  if (!confirm('حذف جميع الوثائق المكررة نهائياً؟ (سيتم الاحتفاظ بالنسخة الأصلية)')) return;
+  btn.disabled = true;
+  btn.textContent = 'جارٍ الحذف...';
+  const res = await fetch('/documents/delete-duplicates', { method: 'POST' });
+  if (res.ok) {
+    const data = await res.json();
+    alert(`تم حذف ${data.deleted} وثيقة مكررة.`);
+    location.reload();
+  } else {
+    alert('فشل الحذف.');
+    btn.disabled = false;
+  }
+}
+</script>
+{% endblock %}

templates/person_detail.html

@@ -101,7 +101,7 @@
   <div class="doc-thumbnails">
     {% for d in documents %}
     <a href="/review/{{ d.id }}" class="doc-thumb">
-      <img src="/uploads/{{ d.image_path }}" alt="وثيقة">
+      <img src="/thumbs/{{ d.image_path }}?size=480" alt="وثيقة" loading="lazy" decoding="async">
       <div class="doc-thumb-info">
         {% if d.request_number %}طلب {{ d.request_number }}{% endif %}
         {% if d.request_date %}<br>{{ d.request_date }}{% endif %}