Fix startup and Gemini provider handling

2026-05-07 12:45:01 +03:00
parent b9b3c5512f
commit 54dd52639d
8 changed files with 189 additions and 132 deletions
@@ -45,7 +45,7 @@ def require_admin(request: Request):

@router.get("/login", response_class=HTMLResponse)
 async def login_get(request: Request):
-    return templates.TemplateResponse(request=request, name="login.html", context={"error": None})
+    return templates.TemplateResponse("login.html", {"request": request, "error": None})

@router.post("/login", response_class=HTMLResponse)
 async def login_post(request: Request, username: str = Form(...), password: str = Form(...)):
@@ -54,9 +54,8 @@ async def login_post(request: Request, username: str = Form(...), password: str

    if is_login_blocked(username, ip):
        return templates.TemplateResponse(
-            request=request,
-            name="login.html",
-            context={"error": "تم حجب محاولات تسجيل الدخول مؤقتاً. حاول بعد 15 دقيقة."},
+            "login.html",
+            {"request": request, "error": "تم حجب محاولات تسجيل الدخول مؤقتاً. حاول بعد 15 دقيقة."},
            status_code=429,
        )

@@ -64,9 +63,8 @@ async def login_post(request: Request, username: str = Form(...), password: str
    if not user or not verify_password(user["password_hash"], password):
        record_login_attempt(username, ip, success=False)
        return templates.TemplateResponse(
-            request=request,
-            name="login.html",
-            context={"error": "Invalid username or password"},
+            "login.html",
+            {"request": request, "error": "Invalid username or password"},
            status_code=401,
        )

@@ -109,9 +107,8 @@ async def logout(request: Request):
 async def users_list(request: Request, current=Depends(require_admin)):
    users = get_all_users()
    return templates.TemplateResponse(
-        request=request,
-        name="users.html",
-        context={"users": users, "current_user": current},
+        "users.html",
+        {"request": request, "users": users, "current_user": current},
    )

@router.post("/users/create")
@@ -128,16 +125,14 @@ async def add_user(
    if len(password) < 8:
        users = get_all_users()
        return templates.TemplateResponse(
-            request=request,
-            name="users.html",
-            context={"users": users, "current_user": current, "error": "Password must be at least 8 characters."},
+            "users.html",
+            {"request": request, "users": users, "current_user": current, "error": "Password must be at least 8 characters."},
        )
    if get_user_by_username(username):
        users = get_all_users()
        return templates.TemplateResponse(
-            request=request,
-            name="users.html",
-            context={"users": users, "current_user": current, "error": f"User '{username}' already exists."},
+            "users.html",
+            {"request": request, "users": users, "current_user": current, "error": f"User '{username}' already exists."},
        )
    create_user(username, password, role=role)
    return RedirectResponse(url="/auth/users", status_code=status.HTTP_303_SEE_OTHER)
@@ -147,9 +142,8 @@ async def remove_user(request: Request, user_id: int, current=Depends(require_ad
    if user_id == current["user_id"]:
        users = get_all_users()
        return templates.TemplateResponse(
-            request=request,
-            name="users.html",
-            context={"users": users, "current_user": current, "error": "لا يمكنك حذف حسابك الخاص."},
+            "users.html",
+            {"request": request, "users": users, "current_user": current, "error": "لا يمكنك حذف حسابك الخاص."},
            status_code=400,
        )
    # Prevent removing the last admin
@@ -157,9 +151,8 @@ async def remove_user(request: Request, user_id: int, current=Depends(require_ad
    if target and target["role"] == "admin" and count_admins() <= 1:
        users = get_all_users()
        return templates.TemplateResponse(
-            request=request,
-            name="users.html",
-            context={"users": users, "current_user": current, "error": "لا يمكن حذف آخر مسؤول في النظام."},
+            "users.html",
+            {"request": request, "users": users, "current_user": current, "error": "لا يمكن حذف آخر مسؤول في النظام."},
            status_code=400,
        )
    delete_user(user_id)
@@ -180,9 +173,8 @@ async def change_role(
    if target and target["role"] == "admin" and role != "admin" and count_admins() <= 1:
        users = get_all_users()
        return templates.TemplateResponse(
-            request=request,
-            name="users.html",
-            context={"users": users, "current_user": current, "error": "لا يمكن تخفيض رتبة آخر مسؤول."},
+            "users.html",
+            {"request": request, "users": users, "current_user": current, "error": "لا يمكن تخفيض رتبة آخر مسؤول."},
            status_code=400,
        )
    set_user_role(user_id, role)
@@ -196,9 +188,9 @@ async def backup_db(request: Request, current=Depends(require_admin)):
    except Exception as e:
        msg = f"Backup failed: {str(e)}"
        return templates.TemplateResponse(
-            request=request,
-            name="users.html",
-            context={
+            "users.html",
+            {
+                "request": request,
                "users": get_all_users(),
                "current_user": current,
                "backup_msg": msg,
@@ -12,6 +12,7 @@ from services.extractor import (
    extract_document,
    get_available_providers,
    get_default_provider,
+    provider_supports_ai_verification,
    verify_page_correlation,
 )
 from services.search_service import normalize_arabic, _normalize_scope
@@ -336,7 +337,8 @@ async def review_document(request: Request, doc_id: int, wait: int = 0):
            "providers": get_available_providers(),
            "current_provider": doc.get("provider") or get_default_provider(),
            "ai_verification_available": any(
-                provider["id"] in {"claude", "gemini"} for provider in get_available_providers()
+                provider_supports_ai_verification(provider["id"])
+                for provider in get_available_providers()
            ),
        },
    )
@@ -9,7 +9,7 @@ from fastapi import APIRouter, File, Form, HTTPException, Request, UploadFile
 from fastapi.responses import JSONResponse, RedirectResponse
 from fastapi.templating import Jinja2Templates

-from config import UPLOAD_DIR
+from config import UPLOAD_DIR, MAX_CONCURRENT_EXTRACTIONS
 from database.connection import get_db
 from services.extractor import extract_document, get_available_providers, get_default_provider
 from services.pdf_handler import pdf_to_images
@@ -17,6 +17,16 @@ from services.pdf_handler import pdf_to_images
 router = APIRouter()
 templates = Jinja2Templates(directory="templates")

+# Semaphore: limits simultaneous AI API calls to avoid quota exhaustion on bulk uploads
+_extraction_semaphore: asyncio.Semaphore | None = None
+
+
+def _get_semaphore() -> asyncio.Semaphore:
+    global _extraction_semaphore
+    if _extraction_semaphore is None:
+        _extraction_semaphore = asyncio.Semaphore(MAX_CONCURRENT_EXTRACTIONS)
+    return _extraction_semaphore
+
 ALLOWED_IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".webp"}
 ALLOWED_PDF_EXTS = {".pdf"}
 ALLOWED_EXTENSIONS = ALLOWED_IMAGE_EXTS | ALLOWED_PDF_EXTS
@@ -91,95 +101,96 @@ def _save_image(file_bytes: bytes, original_name: str) -> str:

 async def _extract_and_save(doc_id: int, image_path: str, provider: str = ""):
    """Background task: call extractor, parse result, update DB."""
-    try:
-        data = await extract_document(image_path, provider=provider)
-        raw_json = json.dumps(data, ensure_ascii=False)
+    async with _get_semaphore():
+        try:
+            data = await extract_document(image_path, provider=provider)
+            raw_json = json.dumps(data, ensure_ascii=False)

-        with get_db() as conn:
-            conn.execute(
-                """UPDATE documents SET status='extracted',
-                   raw_extraction_json=?,
-                   request_number=?,
-                   request_date=?,
-                   applicant_name_raw=?,
-                   request_purpose=?,
-                   data_valid_until=?,
-                   registry_office=?,
-                   owns_properties=?,
-                   declared_property_count=?,
-                   page_info=?,
-                   search_scope=?,
-                   updated_at=CURRENT_TIMESTAMP
-                   WHERE id=?""",
-                (
-                    raw_json,
-                    data.get("request_number"),
-                    data.get("request_date"),
-                    data.get("applicant_name_raw"),
-                    data.get("request_purpose"),
-                    data.get("data_valid_until"),
-                    data.get("registry_office"),
-                    data.get("owns_properties"),
-                    data.get("declared_property_count"),
-                    data.get("page_info"),
-                    data.get("search_scope"),
-                    doc_id,
-                ),
-            )
-
-            for i, prop in enumerate(data.get("properties", [])):
+            with get_db() as conn:
                conn.execute(
-                    """INSERT INTO properties
-                       (document_id, row_order, party_name, property_number,
-                        section, block, real_estate_district, qaza, num_shares, ownership_type)
-                       VALUES (?,?,?,?,?,?,?,?,?,?)""",
+                    """UPDATE documents SET status='extracted',
+                       raw_extraction_json=?,
+                       request_number=?,
+                       request_date=?,
+                       applicant_name_raw=?,
+                       request_purpose=?,
+                       data_valid_until=?,
+                       registry_office=?,
+                       owns_properties=?,
+                       declared_property_count=?,
+                       page_info=?,
+                       search_scope=?,
+                       updated_at=CURRENT_TIMESTAMP
+                       WHERE id=?""",
                    (
-                        doc_id, i,
-                        prop.get("party_name"),
-                        prop.get("property_number"),
-                        prop.get("section"),
-                        prop.get("block"),
-                        prop.get("real_estate_district"),
-                        prop.get("qaza"),
-                        prop.get("num_shares"),
-                        prop.get("ownership_type"),
+                        raw_json,
+                        data.get("request_number"),
+                        data.get("request_date"),
+                        data.get("applicant_name_raw"),
+                        data.get("request_purpose"),
+                        data.get("data_valid_until"),
+                        data.get("registry_office"),
+                        data.get("owns_properties"),
+                        data.get("declared_property_count"),
+                        data.get("page_info"),
+                        data.get("search_scope"),
+                        doc_id,
                    ),
                )

-            # Flag logical duplicates: same request_number + search_scope + page_number
-            req_num = (data.get("request_number") or "").strip()
-            scope = (data.get("search_scope") or "").strip()
-            page_info = (data.get("page_info") or "").strip()
-            if req_num:
-                existing = conn.execute(
-                    """SELECT id FROM documents
-                       WHERE id != ? AND request_number=?
-                         AND COALESCE(search_scope,'')=?
-                         AND COALESCE(page_info,'')=?
-                         AND status IN ('extracted','confirmed')
-                         AND COALESCE(duplicate_dismissed, 0) = 0
-                       ORDER BY id LIMIT 1""",
-                    (doc_id, req_num, scope, page_info),
-                ).fetchone()
-                if existing:
-                    # Only auto-flag if THIS document hasn't itself been dismissed.
-                    self_row = conn.execute(
-                        "SELECT COALESCE(duplicate_dismissed, 0) AS d FROM documents WHERE id=?",
-                        (doc_id,),
-                    ).fetchone()
-                    if not (self_row and self_row["d"]):
-                        conn.execute(
-                            "UPDATE documents SET duplicate_of=? WHERE id=?",
-                            (existing["id"], doc_id),
-                        )
+                for i, prop in enumerate(data.get("properties", [])):
+                    conn.execute(
+                        """INSERT INTO properties
+                           (document_id, row_order, party_name, property_number,
+                            section, block, real_estate_district, qaza, num_shares, ownership_type)
+                           VALUES (?,?,?,?,?,?,?,?,?,?)""",
+                        (
+                            doc_id, i,
+                            prop.get("party_name"),
+                            prop.get("property_number"),
+                            prop.get("section"),
+                            prop.get("block"),
+                            prop.get("real_estate_district"),
+                            prop.get("qaza"),
+                            prop.get("num_shares"),
+                            prop.get("ownership_type"),
+                        ),
+                    )

-    except Exception as e:
-        with get_db() as conn:
-            conn.execute(
-                """UPDATE documents SET status='error', extraction_error=?,
-                   updated_at=CURRENT_TIMESTAMP WHERE id=?""",
-                (str(e), doc_id),
-            )
+                # Flag logical duplicates: same request_number + search_scope + page_number
+                req_num = (data.get("request_number") or "").strip()
+                scope = (data.get("search_scope") or "").strip()
+                page_info = (data.get("page_info") or "").strip()
+                if req_num:
+                    existing = conn.execute(
+                        """SELECT id FROM documents
+                           WHERE id != ? AND request_number=?
+                             AND COALESCE(search_scope,'')=?
+                             AND COALESCE(page_info,'')=?
+                             AND status IN ('extracted','confirmed')
+                             AND COALESCE(duplicate_dismissed, 0) = 0
+                           ORDER BY id LIMIT 1""",
+                        (doc_id, req_num, scope, page_info),
+                    ).fetchone()
+                    if existing:
+                        # Only auto-flag if THIS document hasn't itself been dismissed.
+                        self_row = conn.execute(
+                            "SELECT COALESCE(duplicate_dismissed, 0) AS d FROM documents WHERE id=?",
+                            (doc_id,),
+                        ).fetchone()
+                        if not (self_row and self_row["d"]):
+                            conn.execute(
+                                "UPDATE documents SET duplicate_of=? WHERE id=?",
+                                (existing["id"], doc_id),
+                            )
+
+        except Exception as e:
+            with get_db() as conn:
+                conn.execute(
+                    """UPDATE documents SET status='error', extraction_error=?,
+                       updated_at=CURRENT_TIMESTAMP WHERE id=?""",
+                    (str(e), doc_id),
+                )


@router.get("/")