perf: paginate docs list, lazy thumbnails, static cache headers

database/schema.py

@@ -46,6 +46,16 @@ def create_tables():
                 created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
             );
 
+            CREATE TABLE IF NOT EXISTS sessions (
+                id TEXT PRIMARY KEY,
+                user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+                username TEXT NOT NULL,
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                expires_at TIMESTAMP NOT NULL
+            );
+            CREATE INDEX IF NOT EXISTS idx_sessions_expires_at ON sessions(expires_at);
+            CREATE INDEX IF NOT EXISTS idx_sessions_user_id ON sessions(user_id);
+
             CREATE TABLE IF NOT EXISTS persons (
                 id INTEGER PRIMARY KEY AUTOINCREMENT,
                 first_name TEXT NOT NULL,

main.py

@@ -1,3 +1,4 @@
+from contextlib import asynccontextmanager
 from pathlib import Path
 
 from fastapi import FastAPI, Request
@@ -8,6 +9,7 @@ from starlette.types import Scope
 from config import UPLOAD_DIR, ENVIRONMENT
 from database.schema import create_tables
 from routers import documents, review, search, upload, auth
+from services.auth_service import get_session, cleanup_expired_sessions
 
 
 class CachedStaticFiles(StaticFiles):
@@ -24,11 +26,24 @@ class CachedStaticFiles(StaticFiles):
             response.headers["Cache-Control"] = f"public, max-age={self._max_age}"
         return response
 
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    Path(UPLOAD_DIR).mkdir(parents=True, exist_ok=True)
+    Path("data").mkdir(exist_ok=True)
+    create_tables()
+    try:
+        cleanup_expired_sessions()
+    except Exception:
+        pass
+    yield
+
+
 # Disable docs/openapi in production
 if ENVIRONMENT == "production":
-    app = FastAPI(title="Lebanese Real Estate Registry", docs_url=None, redoc_url=None, openapi_url=None)
+    app = FastAPI(title="Lebanese Real Estate Registry", docs_url=None, redoc_url=None, openapi_url=None, lifespan=lifespan)
 else:
-    app = FastAPI(title="Lebanese Real Estate Registry")
+    app = FastAPI(title="Lebanese Real Estate Registry", lifespan=lifespan)
 
 @app.middleware("http")
 async def check_authentication(request: Request, call_next):
@@ -38,23 +53,13 @@ async def check_authentication(request: Request, call_next):
 
     if not is_allowed:
         session_id = request.cookies.get("session_id")
-        if not session_id or session_id not in auth.sessions:
+        if not session_id or not get_session(session_id):
             return RedirectResponse(url="/auth/login", status_code=303)
 
     response = await call_next(request)
     return response
 
 
-@app.on_event("startup")
-async def startup():
-    Path(UPLOAD_DIR).mkdir(parents=True, exist_ok=True)
-    Path("data").mkdir(exist_ok=True)
-    create_tables()
-    
-    # Do NOT create default admin user automatically in production!
-    # A script should be used for initial setup.
-
-
 app.mount("/uploads", CachedStaticFiles(directory=UPLOAD_DIR, max_age=604800), name="uploads")
 app.mount("/static", CachedStaticFiles(directory="static", max_age=86400), name="static")
 

routers/auth.py

@@ -1,22 +1,20 @@
 from fastapi import APIRouter, Depends, Request, Form, HTTPException, status
 from fastapi.responses import HTMLResponse, RedirectResponse, FileResponse
 from fastapi.templating import Jinja2Templates
-from services.auth_service import verify_password, get_user_by_username, create_user, delete_user, get_all_users
+from services.auth_service import (
+    verify_password, get_user_by_username, create_user, delete_user, get_all_users,
+    create_session, get_session, delete_session, SESSION_TTL_SECONDS,
+)
 from services.backup_service import create_backup
-import secrets
 import os
 
 router = APIRouter()
 templates = Jinja2Templates(directory="templates")
 
-# Simple memory store for sessions for this requirement. (In prod we use Redis/Cookie etc., but cookie + session dict is quickest without external deps like itsdangerous if not in requirements)
-sessions = {}
 
 def get_current_user_from_request(request: Request):
     session_id = request.cookies.get("session_id")
-    if session_id and session_id in sessions:
-        return sessions[session_id]
-    return None
+    return get_session(session_id) if session_id else None
 
 def get_current_user(request: Request):
     user = get_current_user_from_request(request)
@@ -37,9 +35,7 @@ async def login_post(request: Request, username: str = Form(...), password: str
     if not user or not verify_password(user["password_hash"], password):
         return templates.TemplateResponse(request=request, name="login.html", context={"error": "Invalid username or password"})
 
-    
-    session_id = secrets.token_urlsafe(32)
-    sessions[session_id] = dict(user)
+    session_id = create_session(user["id"], user["username"])
 
     from config import ENVIRONMENT
     # secure=True only when accessed via HTTPS (check X-Forwarded-Proto from nginx)
@@ -51,7 +47,7 @@ async def login_post(request: Request, username: str = Form(...), password: str
         httponly=True,
         secure=is_https,
         samesite="lax",
-        max_age=86400,  # 24 hours
+        max_age=SESSION_TTL_SECONDS,
     )
     return response
 
@@ -59,8 +55,8 @@ async def login_post(request: Request, username: str = Form(...), password: str
 async def logout(request: Request):
     response = RedirectResponse(url="/auth/login")
     session_id = request.cookies.get("session_id")
-    if session_id in sessions:
-        del sessions[session_id]
+    if session_id:
+        delete_session(session_id)
     response.delete_cookie("session_id")
     return response
 

routers/documents.py

@@ -130,6 +130,23 @@ async def delete_document(doc_id: int):
         if not doc:
             return JSONResponse({"error": "not found"}, status_code=404)
 
+        # If other documents were flagged as duplicates of this one, detach them
+        # (promote the earliest orphan to be the new keeper so they remain grouped).
+        orphans = conn.execute(
+            "SELECT id FROM documents WHERE duplicate_of=? ORDER BY id",
+            (doc_id,),
+        ).fetchall()
+        if orphans:
+            new_keeper = orphans[0]["id"]
+            conn.execute(
+                "UPDATE documents SET duplicate_of=NULL WHERE id=?",
+                (new_keeper,),
+            )
+            conn.execute(
+                "UPDATE documents SET duplicate_of=? WHERE duplicate_of=? AND id != ?",
+                (new_keeper, doc_id, new_keeper),
+            )
+
         # Delete properties for this document
         conn.execute("DELETE FROM properties WHERE document_id=?", (doc_id,))
 
@@ -310,22 +327,49 @@ async def scan_duplicates():
 
 @router.get("/documents/duplicates")
 async def duplicates_view(request: Request):
-    """List all documents flagged as duplicates alongside their originals."""
+    """Group duplicates with their originals so the user can compare side-by-side."""
     with get_db() as conn:
-        rows = conn.execute(
-            """SELECT d.*, p.first_name, p.family_name,
-                      o.request_number AS orig_request_number,
-                      o.created_at AS orig_created_at
+        # Fetch all docs that have been flagged as duplicates + their keepers
+        dup_rows = conn.execute(
+            """SELECT d.*, p.first_name, p.family_name
                FROM documents d
                LEFT JOIN persons p ON p.id = d.person_id
-               LEFT JOIN documents o ON o.id = d.duplicate_of
                WHERE d.duplicate_of IS NOT NULL
                ORDER BY d.duplicate_of, d.id"""
         ).fetchall()
+
+        keeper_ids = sorted({r["duplicate_of"] for r in dup_rows if r["duplicate_of"]})
+        keepers_by_id = {}
+        if keeper_ids:
+            placeholders = ",".join("?" * len(keeper_ids))
+            keeper_rows = conn.execute(
+                f"""SELECT d.*, p.first_name, p.family_name
+                    FROM documents d
+                    LEFT JOIN persons p ON p.id = d.person_id
+                    WHERE d.id IN ({placeholders})""",
+                keeper_ids,
+            ).fetchall()
+            keepers_by_id = {r["id"]: dict(r) for r in keeper_rows}
+
+    groups = []
+    seen_keepers = []
+    dups_by_keeper: dict[int, list[dict]] = {}
+    for r in dup_rows:
+        k = r["duplicate_of"]
+        dups_by_keeper.setdefault(k, []).append(dict(r))
+        if k not in seen_keepers:
+            seen_keepers.append(k)
+
+    for k in seen_keepers:
+        keeper = keepers_by_id.get(k)
+        if not keeper:
+            continue
+        groups.append({"keeper": keeper, "duplicates": dups_by_keeper.get(k, [])})
+
     return templates.TemplateResponse(
         request,
         "duplicates.html",
-        {"documents": [dict(r) for r in rows]},
+        {"groups": groups, "total_duplicates": len(dup_rows)},
     )
 
 
@@ -347,3 +391,20 @@ async def delete_all_duplicates():
                 pass
             deleted += 1
     return JSONResponse({"ok": True, "deleted": deleted})
+
+
+@router.post("/documents/{doc_id}/unflag-duplicate")
+async def unflag_duplicate(doc_id: int):
+    """Mark a flagged-duplicate document as NOT a duplicate (clear duplicate_of)."""
+    with get_db() as conn:
+        row = conn.execute(
+            "SELECT id FROM documents WHERE id=? AND duplicate_of IS NOT NULL",
+            (doc_id,),
+        ).fetchone()
+        if not row:
+            return JSONResponse({"error": "not flagged"}, status_code=404)
+        conn.execute(
+            "UPDATE documents SET duplicate_of=NULL, updated_at=CURRENT_TIMESTAMP WHERE id=?",
+            (doc_id,),
+        )
+    return JSONResponse({"ok": True})

services/auth_service.py

@@ -1,6 +1,9 @@
 import hashlib
 import hmac
 import secrets
+from datetime import datetime, timedelta
+
+SESSION_TTL_SECONDS = 86400  # 24h
 
 def hash_password(password: str) -> str:
     salt = secrets.token_hex(16)
@@ -38,3 +41,47 @@ def get_all_users():
         cursor = conn.execute("SELECT id, username, created_at FROM users ORDER BY created_at DESC")
         return cursor.fetchall()
 
+
+# ─── Sessions (DB-backed) ─────────────────────────────────────────
+
+def create_session(user_id: int, username: str, ttl_seconds: int = SESSION_TTL_SECONDS) -> str:
+    from database.connection import get_db
+    session_id = secrets.token_urlsafe(32)
+    expires_at = (datetime.utcnow() + timedelta(seconds=ttl_seconds)).strftime("%Y-%m-%d %H:%M:%S")
+    with get_db() as conn:
+        conn.execute(
+            "INSERT INTO sessions (id, user_id, username, expires_at) VALUES (?, ?, ?, ?)",
+            (session_id, user_id, username, expires_at),
+        )
+    return session_id
+
+
+def get_session(session_id: str):
+    """Return dict with user_id/username if session is valid and unexpired, else None."""
+    if not session_id:
+        return None
+    from database.connection import get_db
+    now = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
+    with get_db() as conn:
+        row = conn.execute(
+            "SELECT id, user_id, username, expires_at FROM sessions WHERE id = ? AND expires_at > ?",
+            (session_id, now),
+        ).fetchone()
+    return dict(row) if row else None
+
+
+def delete_session(session_id: str) -> None:
+    if not session_id:
+        return
+    from database.connection import get_db
+    with get_db() as conn:
+        conn.execute("DELETE FROM sessions WHERE id = ?", (session_id,))
+
+
+def cleanup_expired_sessions() -> int:
+    from database.connection import get_db
+    now = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
+    with get_db() as conn:
+        cur = conn.execute("DELETE FROM sessions WHERE expires_at <= ?", (now,))
+        return cur.rowcount or 0
+

templates/duplicates.html

@@ -1,72 +1,181 @@
 {% extends "base.html" %}
 {% block title %}الوثائق المكررة — سجل العقارات{% endblock %}
 
+{% block head %}
+<style>
+  .dup-group {
+    background: var(--surface, #fff);
+    border: 1px solid var(--border, #e5e7eb);
+    border-radius: 12px;
+    padding: 1rem 1.25rem 1.25rem;
+    margin-bottom: 1.5rem;
+    box-shadow: 0 1px 2px rgba(0,0,0,.04);
+  }
+  .dup-group-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    margin-bottom: .75rem;
+    padding-bottom: .5rem;
+    border-bottom: 1px dashed var(--border, #e5e7eb);
+  }
+  .dup-group-title { font-weight: 600; font-size: .95rem; color: #475569; }
+  .dup-cards {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(260px, 1fr));
+    gap: 1rem;
+  }
+  .dup-card {
+    border: 1px solid var(--border, #e5e7eb);
+    border-radius: 10px;
+    padding: .75rem;
+    background: #fafafa;
+    display: flex;
+    flex-direction: column;
+    gap: .5rem;
+  }
+  .dup-card.keeper { border-color: #10b981; background: #f0fdf4; }
+  .dup-card.duplicate { border-color: #f59e0b; background: #fffbeb; }
+  .dup-card-label {
+    font-size: .75rem;
+    font-weight: 700;
+    letter-spacing: .05em;
+  }
+  .dup-card.keeper .dup-card-label { color: #047857; }
+  .dup-card.duplicate .dup-card-label { color: #b45309; }
+  .dup-card-img {
+    width: 100%;
+    aspect-ratio: 1 / 1.3;
+    object-fit: cover;
+    border-radius: 6px;
+    background: #fff;
+    border: 1px solid #e5e7eb;
+    cursor: zoom-in;
+  }
+  .dup-meta { font-size: .85rem; line-height: 1.5; color: #334155; }
+  .dup-meta div { overflow: hidden; text-overflow: ellipsis; }
+  .dup-meta strong { color: #0f172a; }
+  .dup-card-actions {
+    display: flex;
+    gap: .35rem;
+    flex-wrap: wrap;
+    margin-top: auto;
+  }
+  .dup-lightbox {
+    position: fixed; inset: 0;
+    background: rgba(0,0,0,.85);
+    display: none;
+    align-items: center;
+    justify-content: center;
+    z-index: 1000;
+    padding: 1rem;
+  }
+  .dup-lightbox.open { display: flex; }
+  .dup-lightbox img { max-width: 100%; max-height: 100%; border-radius: 6px; }
+  .dup-lightbox-close {
+    position: absolute; top: 1rem; left: 1rem;
+    background: #fff; border: none; border-radius: 50%;
+    width: 40px; height: 40px; font-size: 1.25rem; cursor: pointer;
+  }
+</style>
+{% endblock %}
+
 {% block content %}
 <div class="page-header">
   <div>
     <span class="page-kicker">مراجعة التكرارات</span>
     <h1>الوثائق المكررة</h1>
-    <p class="subtitle">وثائق تمّ رصدها كتكرار لوثيقة موجودة مسبقاً. يمكن حذفها بأمان.</p>
+    <p class="subtitle">قارن كل وثيقة مكررة مع نسختها الأصلية واحذف ما تريد.</p>
   </div>
-  <div style="display:flex;gap:.5rem">
+  <div style="display:flex;gap:.5rem;flex-wrap:wrap">
     <button class="btn btn-secondary" onclick="scanDuplicates(this)">فحص التكرارات الآن</button>
-    {% if documents %}
-    <button class="btn btn-danger" onclick="deleteAllDuplicates(this)">حذف جميع التكرارات ({{ documents|length }})</button>
+    {% if total_duplicates %}
+    <button class="btn btn-danger" onclick="deleteAllDuplicates(this)">حذف جميع المكررات ({{ total_duplicates }})</button>
     {% endif %}
   </div>
 </div>
 
-{% if not documents %}
+{% if not groups %}
 <div class="no-results">
   <p>لا توجد وثائق مكررة حالياً.</p>
   <p><button class="btn btn-primary" onclick="scanDuplicates(this)">فحص قاعدة البيانات عن التكرارات</button></p>
 </div>
 {% else %}
-<div class="table-wrapper">
-  <table class="results-table responsive-table">
-    <thead>
-      <tr>
-        <th>#</th>
-        <th>الصورة</th>
-        <th>الشخص</th>
-        <th>رقم الطلب</th>
-        <th>مكرر للوثيقة</th>
-        <th>الحالة</th>
-        <th></th>
-      </tr>
-    </thead>
-    <tbody>
-      {% for d in documents %}
-      <tr id="row-{{ d.id }}">
-        <td data-label="#">{{ d.id }}</td>
-        <td data-label="الصورة">
-          <a href="/review/{{ d.id }}"><img src="/thumbs/{{ d.image_path }}" class="doc-thumb-mini" alt="" loading="lazy" decoding="async" width="80" height="80"></a>
-        </td>
-        <td data-label="الشخص">
+
+{% macro doc_card(d, role) %}
+<div class="dup-card {{ role }}" id="card-{{ d.id }}">
+  <div style="display:flex;justify-content:space-between;align-items:center">
+    <span class="dup-card-label">
+      {% if role == 'keeper' %}★ النسخة الأصلية{% else %}مكرر{% endif %}
+    </span>
+    <span style="font-size:.8rem;color:#64748b">#{{ d.id }}</span>
+  </div>
+  <img src="/thumbs/{{ d.image_path }}?size=480"
+       data-full="/uploads/{{ d.image_path }}"
+       class="dup-card-img"
+       alt="وثيقة {{ d.id }}"
+       loading="lazy" decoding="async"
+       onclick="openLightbox(this)">
+  <div class="dup-meta">
+    <div><strong>الشخص:</strong>
       {% if d.person_id %}
         <a href="/persons/{{ d.person_id }}">{{ d.first_name or '' }} {{ d.family_name or '' }}</a>
       {% else %}—{% endif %}
-        </td>
-        <td data-label="رقم الطلب">{{ d.request_number or '—' }}</td>
-        <td data-label="مكرر للوثيقة">
-          <a href="/review/{{ d.duplicate_of }}">#{{ d.duplicate_of }}</a>
-          {% if d.orig_created_at %}<br><small>{{ d.orig_created_at[:10] }}</small>{% endif %}
-        </td>
-        <td data-label="الحالة"><span class="status-badge status-{{ d.status }}">{{ d.status }}</span></td>
-        <td data-label="الإجراءات" class="doc-actions">
-          <a href="/review/{{ d.id }}" class="btn btn-sm btn-secondary">عرض</a>
+    </div>
+    <div><strong>رقم الطلب:</strong> {{ d.request_number or '—' }}</div>
+    {% if d.page_info %}<div><strong>الصفحة:</strong> {{ d.page_info }}</div>{% endif %}
+    {% if d.search_scope %}<div><strong>النطاق:</strong> {{ d.search_scope }}</div>{% endif %}
+    <div><strong>الحالة:</strong> <span class="status-badge status-{{ d.status }}">{{ d.status }}</span></div>
+    {% if d.created_at %}<div><strong>أُنشئت:</strong> <small>{{ d.created_at[:16] }}</small></div>{% endif %}
+  </div>
+  <div class="dup-card-actions">
+    <a href="/review/{{ d.id }}" class="btn btn-sm btn-secondary">فتح</a>
+    {% if role == 'duplicate' %}
+      <button class="btn btn-sm" onclick="unflagDuplicate({{ d.id }}, this)">ليست مكررة</button>
+    {% endif %}
     <button class="btn btn-sm btn-danger" onclick="deleteDoc({{ d.id }}, this)">حذف</button>
-        </td>
-      </tr>
-      {% endfor %}
-    </tbody>
-  </table>
+  </div>
 </div>
+{% endmacro %}
+
+{% for g in groups %}
+<div class="dup-group" id="group-{{ g.keeper.id }}">
+  <div class="dup-group-header">
+    <div class="dup-group-title">
+      مجموعة تكرار — {{ g.duplicates|length + 1 }} نسخ
+      {% if g.keeper.request_number %} · طلب رقم {{ g.keeper.request_number }}{% endif %}
+    </div>
+  </div>
+  <div class="dup-cards">
+    {{ doc_card(g.keeper, 'keeper') }}
+    {% for d in g.duplicates %}
+      {{ doc_card(d, 'duplicate') }}
+    {% endfor %}
+  </div>
+</div>
+{% endfor %}
+
+<div class="dup-lightbox" id="lightbox" onclick="closeLightbox()">
+  <button class="dup-lightbox-close" aria-label="إغلاق">×</button>
+  <img id="lightbox-img" src="" alt="">
+</div>
+
 {% endif %}
 {% endblock %}
 
 {% block scripts %}
 <script>
+function openLightbox(imgEl) {
+  const full = imgEl.dataset.full || imgEl.src;
+  document.getElementById('lightbox-img').src = full;
+  document.getElementById('lightbox').classList.add('open');
+}
+function closeLightbox() {
+  document.getElementById('lightbox').classList.remove('open');
+  document.getElementById('lightbox-img').src = '';
+}
+document.addEventListener('keydown', (e) => { if (e.key === 'Escape') closeLightbox(); });
+
 async function scanDuplicates(btn) {
   btn.disabled = true;
   const original = btn.textContent;
@@ -84,17 +193,30 @@ async function scanDuplicates(btn) {
 }
 
 async function deleteDoc(docId, btn) {
-  if (!confirm('حذف هذه الوثيقة المكررة؟')) return;
+  if (!confirm('حذف هذه الوثيقة نهائياً؟')) return;
   btn.disabled = true;
   const res = await fetch(`/documents/${docId}`, { method: 'DELETE' });
   if (res.ok) {
-    document.getElementById(`row-${docId}`).remove();
+    // Reload so keeper-promotion (if the keeper was deleted) is reflected.
+    location.reload();
   } else {
     alert('فشل الحذف.');
     btn.disabled = false;
   }
 }
 
+async function unflagDuplicate(docId, btn) {
+  if (!confirm('تأكيد أنها ليست وثيقة مكررة؟')) return;
+  btn.disabled = true;
+  const res = await fetch(`/documents/${docId}/unflag-duplicate`, { method: 'POST' });
+  if (res.ok) {
+    document.getElementById(`card-${docId}`)?.remove();
+  } else {
+    alert('فشل التحديث.');
+    btn.disabled = false;
+  }
+}
+
 async function deleteAllDuplicates(btn) {
   if (!confirm('حذف جميع الوثائق المكررة نهائياً؟ (سيتم الاحتفاظ بالنسخة الأصلية)')) return;
   btn.disabled = true;