Fix startup and Gemini provider handling
This commit is contained in:
Krikorios
+94
-83
@@ -9,7 +9,7 @@ from fastapi import APIRouter, File, Form, HTTPException, Request, UploadFile
|
||||
from fastapi.responses import JSONResponse, RedirectResponse
|
||||
from fastapi.templating import Jinja2Templates
|
||||
|
||||
from config import UPLOAD_DIR
|
||||
from config import UPLOAD_DIR, MAX_CONCURRENT_EXTRACTIONS
|
||||
from database.connection import get_db
|
||||
from services.extractor import extract_document, get_available_providers, get_default_provider
|
||||
from services.pdf_handler import pdf_to_images
|
||||
@@ -17,6 +17,16 @@ from services.pdf_handler import pdf_to_images
|
||||
router = APIRouter()
|
||||
templates = Jinja2Templates(directory="templates")
|
||||
|
||||
# Semaphore: limits simultaneous AI API calls to avoid quota exhaustion on bulk uploads
|
||||
_extraction_semaphore: asyncio.Semaphore | None = None
|
||||
|
||||
|
||||
def _get_semaphore() -> asyncio.Semaphore:
|
||||
global _extraction_semaphore
|
||||
if _extraction_semaphore is None:
|
||||
_extraction_semaphore = asyncio.Semaphore(MAX_CONCURRENT_EXTRACTIONS)
|
||||
return _extraction_semaphore
|
||||
|
||||
ALLOWED_IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".webp"}
|
||||
ALLOWED_PDF_EXTS = {".pdf"}
|
||||
ALLOWED_EXTENSIONS = ALLOWED_IMAGE_EXTS | ALLOWED_PDF_EXTS
|
||||
@@ -91,95 +101,96 @@ def _save_image(file_bytes: bytes, original_name: str) -> str:
|
||||
|
||||
async def _extract_and_save(doc_id: int, image_path: str, provider: str = ""):
|
||||
"""Background task: call extractor, parse result, update DB."""
|
||||
try:
|
||||
data = await extract_document(image_path, provider=provider)
|
||||
raw_json = json.dumps(data, ensure_ascii=False)
|
||||
async with _get_semaphore():
|
||||
try:
|
||||
data = await extract_document(image_path, provider=provider)
|
||||
raw_json = json.dumps(data, ensure_ascii=False)
|
||||
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""UPDATE documents SET status='extracted',
|
||||
raw_extraction_json=?,
|
||||
request_number=?,
|
||||
request_date=?,
|
||||
applicant_name_raw=?,
|
||||
request_purpose=?,
|
||||
data_valid_until=?,
|
||||
registry_office=?,
|
||||
owns_properties=?,
|
||||
declared_property_count=?,
|
||||
page_info=?,
|
||||
search_scope=?,
|
||||
updated_at=CURRENT_TIMESTAMP
|
||||
WHERE id=?""",
|
||||
(
|
||||
raw_json,
|
||||
data.get("request_number"),
|
||||
data.get("request_date"),
|
||||
data.get("applicant_name_raw"),
|
||||
data.get("request_purpose"),
|
||||
data.get("data_valid_until"),
|
||||
data.get("registry_office"),
|
||||
data.get("owns_properties"),
|
||||
data.get("declared_property_count"),
|
||||
data.get("page_info"),
|
||||
data.get("search_scope"),
|
||||
doc_id,
|
||||
),
|
||||
)
|
||||
|
||||
for i, prop in enumerate(data.get("properties", [])):
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""INSERT INTO properties
|
||||
(document_id, row_order, party_name, property_number,
|
||||
section, block, real_estate_district, qaza, num_shares, ownership_type)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?)""",
|
||||
"""UPDATE documents SET status='extracted',
|
||||
raw_extraction_json=?,
|
||||
request_number=?,
|
||||
request_date=?,
|
||||
applicant_name_raw=?,
|
||||
request_purpose=?,
|
||||
data_valid_until=?,
|
||||
registry_office=?,
|
||||
owns_properties=?,
|
||||
declared_property_count=?,
|
||||
page_info=?,
|
||||
search_scope=?,
|
||||
updated_at=CURRENT_TIMESTAMP
|
||||
WHERE id=?""",
|
||||
(
|
||||
doc_id, i,
|
||||
prop.get("party_name"),
|
||||
prop.get("property_number"),
|
||||
prop.get("section"),
|
||||
prop.get("block"),
|
||||
prop.get("real_estate_district"),
|
||||
prop.get("qaza"),
|
||||
prop.get("num_shares"),
|
||||
prop.get("ownership_type"),
|
||||
raw_json,
|
||||
data.get("request_number"),
|
||||
data.get("request_date"),
|
||||
data.get("applicant_name_raw"),
|
||||
data.get("request_purpose"),
|
||||
data.get("data_valid_until"),
|
||||
data.get("registry_office"),
|
||||
data.get("owns_properties"),
|
||||
data.get("declared_property_count"),
|
||||
data.get("page_info"),
|
||||
data.get("search_scope"),
|
||||
doc_id,
|
||||
),
|
||||
)
|
||||
|
||||
# Flag logical duplicates: same request_number + search_scope + page_number
|
||||
req_num = (data.get("request_number") or "").strip()
|
||||
scope = (data.get("search_scope") or "").strip()
|
||||
page_info = (data.get("page_info") or "").strip()
|
||||
if req_num:
|
||||
existing = conn.execute(
|
||||
"""SELECT id FROM documents
|
||||
WHERE id != ? AND request_number=?
|
||||
AND COALESCE(search_scope,'')=?
|
||||
AND COALESCE(page_info,'')=?
|
||||
AND status IN ('extracted','confirmed')
|
||||
AND COALESCE(duplicate_dismissed, 0) = 0
|
||||
ORDER BY id LIMIT 1""",
|
||||
(doc_id, req_num, scope, page_info),
|
||||
).fetchone()
|
||||
if existing:
|
||||
# Only auto-flag if THIS document hasn't itself been dismissed.
|
||||
self_row = conn.execute(
|
||||
"SELECT COALESCE(duplicate_dismissed, 0) AS d FROM documents WHERE id=?",
|
||||
(doc_id,),
|
||||
).fetchone()
|
||||
if not (self_row and self_row["d"]):
|
||||
conn.execute(
|
||||
"UPDATE documents SET duplicate_of=? WHERE id=?",
|
||||
(existing["id"], doc_id),
|
||||
)
|
||||
for i, prop in enumerate(data.get("properties", [])):
|
||||
conn.execute(
|
||||
"""INSERT INTO properties
|
||||
(document_id, row_order, party_name, property_number,
|
||||
section, block, real_estate_district, qaza, num_shares, ownership_type)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?)""",
|
||||
(
|
||||
doc_id, i,
|
||||
prop.get("party_name"),
|
||||
prop.get("property_number"),
|
||||
prop.get("section"),
|
||||
prop.get("block"),
|
||||
prop.get("real_estate_district"),
|
||||
prop.get("qaza"),
|
||||
prop.get("num_shares"),
|
||||
prop.get("ownership_type"),
|
||||
),
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""UPDATE documents SET status='error', extraction_error=?,
|
||||
updated_at=CURRENT_TIMESTAMP WHERE id=?""",
|
||||
(str(e), doc_id),
|
||||
)
|
||||
# Flag logical duplicates: same request_number + search_scope + page_number
|
||||
req_num = (data.get("request_number") or "").strip()
|
||||
scope = (data.get("search_scope") or "").strip()
|
||||
page_info = (data.get("page_info") or "").strip()
|
||||
if req_num:
|
||||
existing = conn.execute(
|
||||
"""SELECT id FROM documents
|
||||
WHERE id != ? AND request_number=?
|
||||
AND COALESCE(search_scope,'')=?
|
||||
AND COALESCE(page_info,'')=?
|
||||
AND status IN ('extracted','confirmed')
|
||||
AND COALESCE(duplicate_dismissed, 0) = 0
|
||||
ORDER BY id LIMIT 1""",
|
||||
(doc_id, req_num, scope, page_info),
|
||||
).fetchone()
|
||||
if existing:
|
||||
# Only auto-flag if THIS document hasn't itself been dismissed.
|
||||
self_row = conn.execute(
|
||||
"SELECT COALESCE(duplicate_dismissed, 0) AS d FROM documents WHERE id=?",
|
||||
(doc_id,),
|
||||
).fetchone()
|
||||
if not (self_row and self_row["d"]):
|
||||
conn.execute(
|
||||
"UPDATE documents SET duplicate_of=? WHERE id=?",
|
||||
(existing["id"], doc_id),
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""UPDATE documents SET status='error', extraction_error=?,
|
||||
updated_at=CURRENT_TIMESTAMP WHERE id=?""",
|
||||
(str(e), doc_id),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/")
|
||||
|
||||
Reference in New Issue
Block a user