// Reading a driver's documents with Google Cloud Vision, then pulling the // four credential fields out of the text it returns. // // Two things shape this file. First, Lebanese documents are trilingual: a // driving licence carries Arabic and French on the same card, an ID card is // Arabic with Arabic-Indic digits, and a vehicle registration mixes both. So // every label we look for has an Arabic, a French and an English spelling, and // digits are normalised before anything is matched. // // Second, OCR is a suggestion, never an answer. Everything here is best-effort // and each field is returned independently — a licence whose number reads // cleanly but whose expiry is smudged yields the number and leaves expiry // empty. The driver reviews and corrects every field before submitting, and a // human reviewer still approves the profile against the stored scan. Nothing // downstream trusts these values because they came from a scan. const VISION_ENDPOINT = "https://vision.googleapis.com/v1/images:annotate"; export const DOCUMENT_TYPES = ["license", "id", "vehicle_reg"] as const; export type DocumentType = (typeof DOCUMENT_TYPES)[number]; export const isDocumentType = (v: unknown): v is DocumentType => typeof v === "string" && (DOCUMENT_TYPES as readonly string[]).includes(v); /** Which drivers column stores the scan for each document type. */ export const DOCUMENT_COLUMNS: Record = { license: "license_image_url", id: "id_image_url", vehicle_reg: "vehicle_reg_image_url", }; /** * The subset of the onboarding form a scan can fill. Every key is optional: * a field is present only when it was actually read off the document. */ export type ExtractedFields = { license_number?: string; /** Always normalised to YYYY-MM-DD, whatever the card printed. */ license_expiry?: string; national_id?: string; plate_number?: string; car_model?: string; }; // --- Normalisation -------------------------------------------------------- /** * Lebanese ID cards print Arabic-Indic digits (٠١٢…), and Vision returns them * verbatim. Everything downstream — the date parser, the digit-run fallbacks, * the form itself — expects ASCII, so fold them first. Both the Arabic-Indic * (U+0660) and Extended Arabic-Indic (U+06F0, used by some fonts) ranges show * up in practice. */ const toAsciiDigits = (text: string): string => text.replace(/[٠-٩۰-۹]/g, (char) => { const code = char.charCodeAt(0); const base = code >= 0x06f0 ? 0x06f0 : 0x0660; return String(code - base); }); /** * Arabic tashkeel (short-vowel marks) and the tatweel stretcher are decorative * and appear inconsistently in OCR output, so a label match must not depend on * them. Latin text is uppercased so one pattern covers "Permis" and "PERMIS". */ const normalise = (text: string): string => toAsciiDigits(text) .replace(/[ً-ٟـٰ]/g, "") .replace(/[​-‏‪-‮]/g, "") .toUpperCase(); const lines = (text: string): string[] => text .split(/\r?\n/) .map((line) => line.trim()) .filter(Boolean); // --- Dates ---------------------------------------------------------------- const MONTH_NAMES: Record = { JAN: 1, FEV: 2, FEB: 2, MAR: 3, AVR: 4, APR: 4, MAI: 5, MAY: 5, JUN: 6, JUIN: 6, JUL: 7, JUIL: 7, AOU: 8, AUG: 8, SEP: 9, OCT: 10, NOV: 11, DEC: 12, }; const isoDate = (year: number, month: number, day: number): string | null => { if (month < 1 || month > 12 || day < 1 || day > 31) return null; if (year < 1900 || year > 2100) return null; const date = new Date(Date.UTC(year, month - 1, day)); // Rejects the likes of 31/02 that survive the range checks above. if (date.getUTCMonth() !== month - 1 || date.getUTCDate() !== day) return null; return `${year}-${String(month).padStart(2, "0")}-${String(day).padStart(2, "0")}`; }; /** * Every date on a line, normalised to YYYY-MM-DD. * * Lebanese documents print day-first (the French convention), so an ambiguous * pair like 03/04 is read as 3 April. When the second component is above 12 the * card must be month-first after all, so that reading wins instead — which is * how a US-formatted document still parses correctly. */ const datesIn = (line: string): string[] => { const found: string[] = []; // Year-first: 2027-03-14 for (const match of line.matchAll( /\b(\d{4})[-/.](\d{1,2})[-/.](\d{1,2})\b/g, )) { const iso = isoDate(+match[1], +match[2], +match[3]); if (iso) found.push(iso); } // Day-first or month-first: 14/03/2027 for (const match of line.matchAll( /\b(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})\b/g, )) { const [, a, b, year] = match; const iso = +b > 12 ? isoDate(+year, +a, +b) : isoDate(+year, +b, +a); if (iso) found.push(iso); } // Spelled-out month: 14 MAR 2027 for (const match of line.matchAll( /\b(\d{1,2})\s+([A-Z]{3,4})\.?\s+(\d{4})\b/g, )) { const month = MONTH_NAMES[match[2]]; const iso = month ? isoDate(+match[3], month, +match[1]) : null; if (iso) found.push(iso); } return found; }; // Label vocabularies. Arabic first because that is what an ID card leads with. const EXPIRY_LABELS = /صلاحية|الصلاحية|تنتهي|انتهاء|ينتهي|EXPIR|VALABLE|VALIDIT|VALID|JUSQU|UNTIL/; const ISSUE_LABELS = /اصدار|الاصدار|تاريخ الاصدار|DELIVR|ISSUE|ISSUED|EMIS|EMISSION/; const BIRTH_LABELS = /ولادة|الولادة|مواليد|NAISSANCE|BIRTH|NE LE|DOB/; /** * The expiry date, which is the one date on a licence we actually want. * * A licence shows three dates — birth, issue, expiry — and picking the wrong * one fails the driver's submission on a date they never typed. So a labelled * expiry wins outright. Failing that, dates sitting on a birth or issue line * are excluded, along with any the caller already identified by field code, * and the latest remaining future date is taken — expiry is the only one of * the three that can be in the future. */ const findExpiry = ( docLines: string[], exclude: Set = new Set(), ): string | undefined => { const today = new Date().toISOString().slice(0, 10); const unlabelled: string[] = []; for (const line of docLines) { const onLine = datesIn(line).filter((date) => !exclude.has(date)); if (onLine.length === 0) continue; if (EXPIRY_LABELS.test(line)) { // A line reading "issued 14/03/2022 expires 14/03/2027" carries both, and // the later one is the expiry. const future = onLine.filter((date) => date > today).sort(); if (future.length > 0) return future[future.length - 1]; return onLine.sort()[onLine.length - 1]; } if (ISSUE_LABELS.test(line) || BIRTH_LABELS.test(line)) continue; unlabelled.push(...onLine); } const future = unlabelled.filter((date) => date > today).sort(); return future.length > 0 ? future[future.length - 1] : undefined; }; /** * Value printed against a numbered field code. * * The Lebanese licence is an EU-format card (Directive 2006/126/EC), which * means its fields are identified by a printed number rather than a word: 1 is * the surname, 2 the given names, 3 the date of birth, 4a the issue date, 4b * the expiry, 4c the issuing authority, 5 the licence number. Reading those * codes is far more reliable than hunting for "expiry" in three languages, * because the card never prints the word in any of them — the only prose on it * is the "PERMIS DE CONDUIRE / DRIVING LICENSE" title. * * The value normally sits on the same line as its code; when Vision splits the * label column from the value column, it lands on the next line instead, so * both layouts are handled. * * `shape` is what makes the second layout safe. Reading a card whose codes are * stacked ("1 / 2 / 3 / 4a / 4b / 5") followed by the values in their own * block, "the line after code 5" is the *first* value, not the fifth — on the * sample licence that is the surname. Requiring the value to look like the * field it claims to be rejects that mismatch and lets the caller fall through * to a fallback that gets it right. */ const ANY_FIELD_CODE = /^\d{1,2}[ABCD]?[.):]?$/; const numberedField = ( docLines: string[], code: string, shape?: RegExp, ): string | undefined => { // The leading (^|\s) is what stops code "5" matching inside "15." and code // "3" matching inside "13B" — both of which are printed on this card. const inline = new RegExp(`(?:^|\\s)${code}[.):\\s]\\s*(\\S.*)$`); const bare = new RegExp(`^${code}[.):]?$`); const fits = (value: string) => value.length > 0 && (!shape || shape.test(value)); for (let index = 0; index < docLines.length; index += 1) { const match = docLines[index].match(inline); const sameLine = match?.[1]?.trim(); if (sameLine && fits(sameLine)) return sameLine; if (!bare.test(docLines[index])) continue; const next = docLines[index + 1]?.trim(); // A code followed by another code is a label column; there is no value // there to take. if (next && !ANY_FIELD_CODE.test(next) && fits(next)) return next; } return undefined; }; /** The single date in a numbered field's value, if it holds one. */ const numberedDate = (docLines: string[], code: string): string | undefined => { const value = numberedField( docLines, code, /\d{1,4}[-/.]\d{1,2}[-/.]\d{2,4}/, ); return value ? datesIn(value)[0] : undefined; }; // --- Field extraction ----------------------------------------------------- /** * OCR routinely drops the separator between a label and its value, so a capture * can arrive as "N 123456" or with trailing label text from the next column. * Keep the leading run of value-shaped characters and drop the rest. */ const cleanValue = (raw: string, allowed: RegExp): string | undefined => { const value = raw .trim() .replace(/^[:.\-–—\s]+/, "") .split(/\s{2,}/)[0] .trim(); const kept = value .split("") .filter((char) => allowed.test(char)) .join("") .trim(); return kept.length >= 3 ? kept : undefined; }; /** First capture across a list of patterns, tried in order of confidence. */ const firstMatch = ( docLines: string[], patterns: RegExp[], allowed: RegExp, ): string | undefined => { for (const pattern of patterns) { for (const line of docLines) { const match = line.match(pattern); const value = match?.[1] ? cleanValue(match[1], allowed) : undefined; if (value) return value; } } return undefined; }; /** * Last resort when no label was recognised: the longest plausible run of * digits on the card. Dates are stripped first, otherwise "14/03/2027" reads * as an eight-digit licence number. */ const longestDigitRun = ( docLines: string[], min: number, max: number, ): string | undefined => { let best: string | undefined; for (const line of docLines) { const withoutDates = line .replace(/\b\d{1,4}[-/.]\d{1,2}[-/.]\d{2,4}\b/g, " ") .replace(/\b(19|20)\d{2}\b/g, " "); for (const match of withoutDates.matchAll(/\d[\d\s-]{2,}\d/g)) { const digits = match[0].replace(/[\s-]/g, ""); if (digits.length < min || digits.length > max) continue; if (!best || digits.length > best.length) best = digits; } } return best; }; const ALPHANUMERIC = /[A-Z0-9/-]/; const DIGITS_ONLY = /[0-9]/; /** Lebanese plates pair digits with a letter group, Arabic or Latin. */ const PLATE_CHARS = /[A-Z0-9ء-ي/-]/; const MODEL_CHARS = /[A-Z0-9 .-]/; const extractLicense = (docLines: string[]): ExtractedFields => { // Field 5 is the licence number on the EU-format card, and it is by far the // most reliable read — so it is tried before any worded label. The word // patterns cover older Lebanese licences that predate the numbered layout, // and the digit-run fallback covers a card whose codes didn't survive OCR. // A licence number is a run of digits, so requiring some is what keeps a // stacked-label card from handing back the holder's surname here. const field5 = numberedField(docLines, "5", /\d{3,}/); const license_number = (field5 ? cleanValue(field5, ALPHANUMERIC) : undefined) ?? firstMatch( docLines, [ /(?:رقم\s*(?:الرخصة|الاجازة|الرخصه)?|PERMIS\s*(?:DE\s*CONDUIRE\s*)?N|N[°ºO]\s*(?:DE\s*)?PERMIS|LICEN[CS]E\s*(?:NO|NUMBER|N[°ºO]))\s*[:.\-]?\s*([A-Z0-9][A-Z0-9/\- ]{3,19})/, /\bN[°ºO]\s*[:.\-]?\s*([A-Z0-9][A-Z0-9/\- ]{4,19})/, ], ALPHANUMERIC, ) ?? longestDigitRun(docLines, 5, 15); // 3 is the date of birth and 4a the date of issue. Naming them explicitly // does double duty: 4b gives the expiry outright, and knowing the other two // keeps them out of the fallback, which would otherwise be free to mistake a // recent issue date for an expiry. const birth = numberedDate(docLines, "3"); const issued = numberedDate(docLines, "4A"); const expires = numberedDate(docLines, "4B"); const excluded = new Set([birth, issued].filter(Boolean) as string[]); // A card that reads 4b but whose expiry has already passed is a real answer, // not a misread — surface it so the driver sees why the form rejects it, // rather than silently leaving the field blank. const license_expiry = expires ?? findExpiry(docLines, excluded); // A Lebanese licence carries the holder's register number too, but only // behind an explicit label — a bare digit run on a licence is far more // likely to be the licence number itself. const national_id = firstMatch( docLines, [ /(?:رقم\s*(?:الهوية|السجل)|REGISTRE|SEJEL|ID\s*(?:NO|NUMBER)|IDENTITY\s*(?:NO|NUMBER))\s*[:.\-]?\s*([0-9][0-9\- ]{4,19})/, ], DIGITS_ONLY, ); return { license_number, license_expiry, national_id }; }; const extractId = (docLines: string[]): ExtractedFields => ({ national_id: firstMatch( docLines, [ /(?:رقم\s*(?:الهوية|السجل|البطاقة)|N[°ºO]\s*(?:DE\s*)?(?:CARTE|REGISTRE)|REGISTRE|SEJEL|ID\s*(?:NO|NUMBER)|IDENTITY\s*(?:NO|NUMBER))\s*[:.\-]?\s*([0-9][0-9\- ]{4,19})/, /\bرقم\s*[:.\-]?\s*([0-9][0-9\- ]{5,19})/, ], DIGITS_ONLY, ) ?? longestDigitRun(docLines, 6, 14), }); const extractVehicleRegistration = (docLines: string[]): ExtractedFields => { const plate_number = firstMatch( docLines, [ /(?:رقم\s*(?:اللوحة|السيارة)|اللوحة|PLAQUE|IMMATRICULATION|PLATE\s*(?:NO|NUMBER)?|REGISTRATION\s*(?:NO|NUMBER)?)\s*[:.\-]?\s*([0-9ء-يA-Z][0-9A-Zء-ي/\- ]{2,14})/, // Unlabelled but unmistakable: digits, a slash, then the letter group. /\b(\d{1,7}\s*\/\s*[A-Zء-ي]{1,3})\b/, ], PLATE_CHARS, ) ?? undefined; const car_model = firstMatch( docLines, [ /(?:نوع\s*(?:السيارة|المركبة)?|الطراز|MARQUE(?:\s*ET\s*TYPE)?|MODELE|MODÈLE|MAKE|MODEL)\s*[:.\-]?\s*([A-Z][A-Z0-9 .-]{2,29})/, ], MODEL_CHARS, ); return { plate_number, car_model }; }; /** Drops keys whose value came back empty so callers can spread the result. */ const compact = (fields: ExtractedFields): ExtractedFields => Object.fromEntries( Object.entries(fields).filter(([, value]) => Boolean(value)), ) as ExtractedFields; /** Pulls the credential fields out of already-recognised document text. */ export const parseDocumentText = ( text: string, docType: DocumentType, ): ExtractedFields => { const docLines = lines(normalise(text)); if (docLines.length === 0) return {}; switch (docType) { case "license": return compact(extractLicense(docLines)); case "id": return compact(extractId(docLines)); case "vehicle_reg": return compact(extractVehicleRegistration(docLines)); } }; // --- Google Cloud Vision -------------------------------------------------- export class OcrUnavailableError extends Error {} type VisionResponse = { responses?: { fullTextAnnotation?: { text?: string }; error?: { message?: string }; }[]; }; /** * Runs Vision's document OCR over a scan and returns the recognised text. * * DOCUMENT_TEXT_DETECTION (rather than plain TEXT_DETECTION) is the dense-text * model: it keeps the line structure of a card, which is what every label * pattern above depends on. The language hints are the three that appear on * Lebanese documents — without them Vision often transliterates Arabic instead * of reading it. */ export const recogniseDocument = async (image: Buffer): Promise => { const key = process.env.GOOGLE_VISION_API_KEY; if (!key) { throw new OcrUnavailableError("GOOGLE_VISION_API_KEY is not configured."); } let response: Response; try { response = await fetch( `${VISION_ENDPOINT}?key=${encodeURIComponent(key)}`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ requests: [ { image: { content: image.toString("base64") }, features: [{ type: "DOCUMENT_TEXT_DETECTION", maxResults: 1 }], imageContext: { languageHints: ["ar", "fr", "en"] }, }, ], }), // A driver is watching a spinner; failing over to manual entry beats // holding the screen while Vision is slow. signal: AbortSignal.timeout(20_000), }, ); } catch (error) { throw new OcrUnavailableError( `Vision request failed: ${(error as Error).message}`, ); } if (!response.ok) { const detail = await response.text().catch(() => ""); throw new OcrUnavailableError( `Vision responded ${response.status}: ${detail.slice(0, 200)}`, ); } const body = (await response.json()) as VisionResponse; const result = body.responses?.[0]; // Vision reports per-image failures inside a 200 response, so the status // code alone does not tell you the scan was read. if (result?.error?.message) { throw new OcrUnavailableError(`Vision error: ${result.error.message}`); } return result?.fullTextAnnotation?.text ?? ""; };