I think this one was for the quote description field - but we got it wrong so we need to seatch and find the correct field......

2026-05-18 11:54:57 +09:30
parent 069bf11ec2
commit 9dde6b28a1
3 changed files with 283 additions and 14 deletions
@@ -134,6 +134,55 @@ def update_job_description(session: requests.Session, job_uuid: str, payload: di
        raise RuntimeError(f"Job description update failed: HTTP {response.status_code} :: {response.text[:1000]}")


+def extract_company_name(job: Dict[str, Any]) -> str:
+    related = job.get("related")
+    if isinstance(related, dict):
+        company = related.get("company")
+        if isinstance(company, dict):
+            company_name = clean_text(company.get("name"))
+            if company_name:
+                return company_name
+    company = job.get("company")
+    if isinstance(company, dict):
+        company_name = clean_text(company.get("name"))
+        if company_name:
+            return company_name
+    return first_text(job.get("company_name"), job.get("customer_name"))
+
+
+def upsert_job_metadata(conn: sqlite3.Connection, *, job_uuid: str, job: Dict[str, Any], source: str) -> None:
+    job_uuid = clean_text(job_uuid or job.get("uuid"))
+    if not job_uuid:
+        return
+    now = utc_now()
+    conn.execute(
+        """
+        INSERT INTO job_metadata (
+            job_uuid, generated_job_id, job_address, company_name, raw_json,
+            first_seen_at, last_seen_at, source
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+        ON CONFLICT(job_uuid) DO UPDATE SET
+            generated_job_id = excluded.generated_job_id,
+            job_address = excluded.job_address,
+            company_name = excluded.company_name,
+            raw_json = excluded.raw_json,
+            last_seen_at = excluded.last_seen_at,
+            source = excluded.source
+        """,
+        (
+            job_uuid,
+            clean_text(job.get("generated_job_id")),
+            format_job_address(job),
+            extract_company_name(job),
+            json.dumps(job, ensure_ascii=False, sort_keys=True),
+            now,
+            now,
+            source,
+        ),
+    )
+    conn.commit()
+
+
 def create_job_material(session: requests.Session, payload: dict) -> str:
    response = session.post(f"{BASE_URL}/jobmaterial.json", json=payload, timeout=REQUEST_TIMEOUT)
    if not response.ok:
@@ -187,6 +236,21 @@ def get_conn(db_path: Path = POLL_DB_PATH) -> sqlite3.Connection:


 def init_apply_tables(conn: sqlite3.Connection) -> None:
+    conn.execute(
+        """
+        CREATE TABLE IF NOT EXISTS job_metadata (
+            job_uuid TEXT PRIMARY KEY,
+            generated_job_id TEXT,
+            job_address TEXT,
+            company_name TEXT,
+            raw_json TEXT NOT NULL,
+            first_seen_at TEXT NOT NULL,
+            last_seen_at TEXT NOT NULL,
+            source TEXT NOT NULL
+        )
+        """
+    )
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_job_metadata_generated_job_id ON job_metadata(generated_job_id)")
    conn.execute(
        """
        CREATE TABLE IF NOT EXISTS quote_template_apply_runs (
@@ -467,6 +531,8 @@ def main() -> int:

            quote_description_source = clean_text(quote["description"])
            job_details = retrieve_job(session, job_uuid) if quote_description_source else {}
+            if job_details:
+                upsert_job_metadata(conn, job_uuid=job_uuid, job=job_details, source=mode)
            job_update_payload = build_job_update_payload(quote_description_source, job_details)
            job_update_record_payload = {
                "endpoint": f"/job/{job_uuid}.json",
@@ -144,6 +144,21 @@ def init_db(db_path: Path) -> None:
            )
            """
        )
+        conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS job_metadata (
+                job_uuid TEXT PRIMARY KEY,
+                generated_job_id TEXT,
+                job_address TEXT,
+                company_name TEXT,
+                raw_json TEXT NOT NULL,
+                first_seen_at TEXT NOT NULL,
+                last_seen_at TEXT NOT NULL,
+                source TEXT NOT NULL
+            )
+            """
+        )
+        conn.execute("CREATE INDEX IF NOT EXISTS idx_job_metadata_generated_job_id ON job_metadata(generated_job_id)")
        conn.commit()


@@ -205,6 +220,109 @@ def fetch_form_responses(
    return response.status_code, data, filter_expr


+def retrieve_job(
+    *,
+    api_key: str,
+    base_url: str,
+    job_uuid: str,
+    timeout: int,
+) -> Dict[str, Any]:
+    response = requests.get(
+        f"{base_url.rstrip('/')}/job/{job_uuid}.json",
+        headers={"X-Api-Key": api_key, "Accept": "application/json"},
+        timeout=timeout,
+    )
+    if not response.ok:
+        raise RuntimeError(f"Job retrieve failed for {job_uuid}: HTTP {response.status_code}: {response.text[:1000]}")
+    data = response.json()
+    if not isinstance(data, dict):
+        raise RuntimeError(f"Job retrieve expected object response, got {type(data).__name__}")
+    return data
+
+
+def clean_text(value: Any) -> str:
+    if value is None:
+        return ""
+    return str(value).replace("\r\n", "\n").replace("\r", "\n").strip()
+
+
+def first_text(*values: Any) -> str:
+    for value in values:
+        text = clean_text(value)
+        if text:
+            return text
+    return ""
+
+
+def format_job_address(job: Dict[str, Any]) -> str:
+    direct = first_text(
+        job.get("job_address"),
+        job.get("site_address"),
+        job.get("address"),
+        job.get("location_address"),
+        job.get("billing_address"),
+    )
+    if direct:
+        return direct
+
+    parts = [
+        first_text(job.get("street"), job.get("street_address"), job.get("address_1"), job.get("address1")),
+        first_text(job.get("suburb"), job.get("city")),
+        first_text(job.get("state")),
+        first_text(job.get("postcode"), job.get("postal_code"), job.get("zip")),
+    ]
+    return " ".join(part for part in parts if part)
+
+
+def extract_company_name(job: Dict[str, Any]) -> str:
+    related = job.get("related")
+    if isinstance(related, dict):
+        company = related.get("company")
+        if isinstance(company, dict):
+            company_name = clean_text(company.get("name"))
+            if company_name:
+                return company_name
+    company = job.get("company")
+    if isinstance(company, dict):
+        company_name = clean_text(company.get("name"))
+        if company_name:
+            return company_name
+    return first_text(job.get("company_name"), job.get("customer_name"))
+
+
+def upsert_job_metadata(conn: sqlite3.Connection, *, job_uuid: str, job: Dict[str, Any], now: str, source: str) -> None:
+    job_uuid = clean_text(job_uuid or job.get("uuid"))
+    if not job_uuid:
+        return
+
+    values = (
+        job_uuid,
+        clean_text(job.get("generated_job_id")),
+        format_job_address(job),
+        extract_company_name(job),
+        json.dumps(job, ensure_ascii=False, sort_keys=True),
+        now,
+        now,
+        source,
+    )
+    conn.execute(
+        """
+        INSERT INTO job_metadata (
+            job_uuid, generated_job_id, job_address, company_name, raw_json,
+            first_seen_at, last_seen_at, source
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+        ON CONFLICT(job_uuid) DO UPDATE SET
+            generated_job_id = excluded.generated_job_id,
+            job_address = excluded.job_address,
+            company_name = excluded.company_name,
+            raw_json = excluded.raw_json,
+            last_seen_at = excluded.last_seen_at,
+            source = excluded.source
+        """,
+        values,
+    )
+
+
 def insert_or_update_raw(
    conn: sqlite3.Connection,
    row: Dict[str, Any],
@@ -457,6 +575,7 @@ def main() -> int:

        inserted = updated = quote_matches = newly_queued = 0
        now = utc_now()
+        fetched_job_uuids = set()
        if conn is not None:
            for row in rows:
                was_inserted, is_quote = insert_or_update_raw(
@@ -469,6 +588,26 @@ def main() -> int:
                updated += 0 if was_inserted else 1
                if is_quote:
                    quote_matches += 1
+                    job_uuid = clean_text(row.get("regarding_object_uuid"))
+                    if job_uuid and job_uuid not in fetched_job_uuids:
+                        try:
+                            job = retrieve_job(
+                                api_key=api_key,
+                                base_url=args.base_url,
+                                job_uuid=job_uuid,
+                                timeout=args.timeout,
+                            )
+                            upsert_job_metadata(conn, job_uuid=job_uuid, job=job, now=now, source="formresponse_poll")
+                            fetched_job_uuids.add(job_uuid)
+                        except Exception as exc:
+                            # Polling/parsing should still proceed if job metadata enrichment fails.
+                            print(
+                                json.dumps(
+                                    {"warning": "job_metadata_fetch_failed", "job_uuid": job_uuid, "error": str(exc)},
+                                    ensure_ascii=False,
+                                ),
+                                file=sys.stderr,
+                            )
                    if parse_and_store_quote_response(
                        conn,
                        row,
@@ -68,6 +68,7 @@ def html_page(title: str, body: str) -> HTMLResponse:
      code, pre { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }
      pre { white-space: pre-wrap; word-break: break-word; background: #0f172a; color: #e2e8f0; padding: 14px; border-radius: 10px; overflow-x: auto; }
      .pill { display: inline-block; padding: 2px 8px; border-radius: 999px; background: #e0f2fe; color: #075985; font-size: 0.85rem; margin: 2px 4px 2px 0; }
+      .job-id { font-weight: 700; color: #111827; }
      .section { margin: 24px 0; }
      .toolbar { background: white; border: 1px solid #e5e7eb; border-radius: 10px; padding: 12px; margin-bottom: 16px; }
      input[type='text'] { padding: 8px; width: 280px; max-width: 100%; }
@@ -139,6 +140,63 @@ def link_with_params(path, **params):
    return f"{path}?{urlencode(filtered)}" if filtered else path


+def resolve_generated_job_id(job_uuid: str) -> str:
+    job_uuid = str(job_uuid or "").strip()
+    if not job_uuid:
+        return ""
+
+    try:
+        with closing(get_poll_conn()) as conn:
+            row = conn.execute(
+                "select generated_job_id from job_metadata where job_uuid = ?",
+                (job_uuid,),
+            ).fetchone()
+            if row and row["generated_job_id"]:
+                return str(row["generated_job_id"])
+    except sqlite3.Error:
+        pass
+
+    try:
+        with closing(get_conn()) as conn:
+            row = conn.execute(
+                """
+                with jobs as (
+                    select
+                        json_extract(payload_json, '$.data.uuid') as job_uuid,
+                        json_extract(payload_json, '$.data.generated_job_id') as generated_job_id,
+                        received_at
+                    from webhook_events
+                    where json_extract(payload_json, '$.data.generated_job_id') is not null
+                    union all
+                    select
+                        json_extract(payload_json, '$.related.job.uuid') as job_uuid,
+                        json_extract(payload_json, '$.related.job.generated_job_id') as generated_job_id,
+                        received_at
+                    from webhook_form_responses
+                    where json_extract(payload_json, '$.related.job.generated_job_id') is not null
+                )
+                select generated_job_id
+                from jobs
+                where job_uuid = ?
+                  and generated_job_id is not null
+                order by received_at desc
+                limit 1
+                """,
+                (job_uuid,),
+            ).fetchone()
+            if row and row["generated_job_id"]:
+                return str(row["generated_job_id"])
+    except sqlite3.Error:
+        pass
+
+    return ""
+
+
+def job_id_html(job_uuid: str) -> str:
+    generated_job_id = resolve_generated_job_id(job_uuid)
+    return f"<span class='job-id'>{escape(generated_job_id)}</span>" if generated_job_id else ""
+
+
@app.get("/health")
 def health():
    return {"ok": True, "db_path": DB_PATH, "state_db_path": STATE_DB_PATH, "poll_db_path": POLL_DB_PATH}
@@ -459,6 +517,7 @@ def list_generated_materials(page: int = Query(1, ge=1)):
        table_rows.append(
            f"<tr>"
            f"<td><a href='/generated-materials/{row['id']}'>{row['id']}</a></td>"
+            f"<td>{job_id_html(row['job_uuid'])}</td>"
            f"<td>{escape(row['job_uuid'] or '')}</td>"
            f"<td>{escape(row['form_response_uuid'] or '')}</td>"
            f"<td>{escape(row['job_material_uuid'] or '')}</td>"
@@ -470,8 +529,8 @@ def list_generated_materials(page: int = Query(1, ge=1)):

    body = f"""
    <table>
-      <thead><tr><th>ID</th><th>Job UUID</th><th>Form response UUID</th><th>Job material UUID</th><th>Kind</th><th>Source question</th><th>Updated</th></tr></thead>
-      <tbody>{''.join(table_rows) or "<tr><td colspan='7'>No rows found.</td></tr>"}</tbody>
+      <thead><tr><th>ID</th><th>Job ID</th><th>Job UUID</th><th>Form response UUID</th><th>Job material UUID</th><th>Kind</th><th>Source question</th><th>Updated</th></tr></thead>
+      <tbody>{''.join(table_rows) or "<tr><td colspan='8'>No rows found.</td></tr>"}</tbody>
    </table>
    <div class='pagination'>
      {f"<a href='{link_with_params('/generated-materials', page=page-1)}'>← Prev</a>" if page > 1 else ''}
@@ -495,6 +554,7 @@ def generated_material_detail(row_id: int):
    body = f"""
    <div class='card summary-grid'>
      <div><strong>ID</strong></div><div>{row['id']}</div>
+      <div><strong>Job ID</strong></div><div>{job_id_html(row['job_uuid'])}</div>
      <div><strong>Job UUID</strong></div><div>{escape(row['job_uuid'] or '')}</div>
      <div><strong>Form response UUID</strong></div><div>{escape(row['form_response_uuid'] or '')}</div>
      <div><strong>Job material UUID</strong></div><div>{escape(row['job_material_uuid'] or '')}</div>
@@ -585,15 +645,15 @@ def list_remote_existing_incidents(page: int = Query(1, ge=1)):
            f"<tr><td><a href='/poll/remote-existing-incidents/{row['id']}'>{row['id']}</a></td>"
            f"<td>{escape(row['detected_at'] or '')}</td><td>{escape(row['action'] or '')}</td>"
            f"<td><a href='/poll/quote-template/{escape(row['form_response_uuid'])}'>{escape(row['form_response_uuid'])}</a></td>"
-            f"<td>{escape(row['job_uuid'] or '')}</td><td>{run_link}</td>"
+            f"<td>{job_id_html(row['job_uuid'])}</td><td>{escape(row['job_uuid'] or '')}</td><td>{run_link}</td>"
            f"<td>{row['desired_count']}</td><td>{row['remote_count']}</td><td>{row['remote_active_count']}</td>"
            f"<td>{escape((row['reason'] or '')[:180])}</td></tr>"
        )

    body = f"""
    <table>
-      <thead><tr><th>ID</th><th>Detected</th><th>Action</th><th>Form response UUID</th><th>Job UUID</th><th>Apply run</th><th>Desired</th><th>Remote rows</th><th>Active</th><th>Reason</th></tr></thead>
-      <tbody>{''.join(table_rows) or "<tr><td colspan='10'>No remote-existing incidents found.</td></tr>"}</tbody>
+      <thead><tr><th>ID</th><th>Detected</th><th>Action</th><th>Form response UUID</th><th>Job ID</th><th>Job UUID</th><th>Apply run</th><th>Desired</th><th>Remote rows</th><th>Active</th><th>Reason</th></tr></thead>
+      <tbody>{''.join(table_rows) or "<tr><td colspan='11'>No remote-existing incidents found.</td></tr>"}</tbody>
    </table>
    <div class='pagination'>
      {f"<a href='{link_with_params('/poll/remote-existing-incidents', page=page-1)}'>← Prev</a>" if page > 1 else ''}
@@ -631,6 +691,7 @@ def remote_existing_incident_detail(incident_id: int):
      <div><strong>Detected</strong></div><div>{escape(row['detected_at'] or '')}</div>
      <div><strong>Action</strong></div><div>{escape(row['action'] or '')}</div>
      <div><strong>Form response UUID</strong></div><div><a href='/poll/quote-template/{escape(row['form_response_uuid'])}'>{escape(row['form_response_uuid'])}</a></div>
+      <div><strong>Job ID</strong></div><div>{job_id_html(row['job_uuid'])}</div>
      <div><strong>Job UUID</strong></div><div>{escape(row['job_uuid'] or '')}</div>
      <div><strong>Apply run</strong></div><div>{run_link}</div>
      <div><strong>Desired rows</strong></div><div>{row['desired_count']}</div>
@@ -668,14 +729,14 @@ def list_apply_runs(page: int = Query(1, ge=1)):
            f"<tr><td><a href='/poll/apply-runs/{row['id']}'>{row['id']}</a></td>"
            f"<td>{escape(row['mode'] or '')}</td><td>{escape(row['status'] or '')}</td>"
            f"<td><a href='/poll/quote-template/{escape(row['form_response_uuid'])}'>{escape(row['form_response_uuid'])}</a></td>"
-            f"<td>{escape(row['job_uuid'] or '')}</td><td>{escape(row['started_at'] or '')}</td><td>{escape(row['finished_at'] or '')}</td>"
+            f"<td>{job_id_html(row['job_uuid'])}</td><td>{escape(row['job_uuid'] or '')}</td><td>{escape(row['started_at'] or '')}</td><td>{escape(row['finished_at'] or '')}</td>"
            f"<td>{row['desired_count']}</td><td>{row['created_count']}</td><td>{escape((row['error'] or '')[:160])}</td></tr>"
        )

    body = f"""
    <table>
-      <thead><tr><th>ID</th><th>Mode</th><th>Status</th><th>Form response UUID</th><th>Job UUID</th><th>Started</th><th>Finished</th><th>Desired</th><th>Created</th><th>Error</th></tr></thead>
-      <tbody>{''.join(table_rows) or "<tr><td colspan='10'>No apply runs found yet.</td></tr>"}</tbody>
+      <thead><tr><th>ID</th><th>Mode</th><th>Status</th><th>Form response UUID</th><th>Job ID</th><th>Job UUID</th><th>Started</th><th>Finished</th><th>Desired</th><th>Created</th><th>Error</th></tr></thead>
+      <tbody>{''.join(table_rows) or "<tr><td colspan='11'>No apply runs found yet.</td></tr>"}</tbody>
    </table>
    <div class='pagination'>
      {f"<a href='{link_with_params('/poll/apply-runs', page=page-1)}'>← Prev</a>" if page > 1 else ''}
@@ -727,6 +788,7 @@ def apply_run_detail(run_id: int):
      <div><strong>Mode</strong></div><div>{escape(run['mode'] or '')}</div>
      <div><strong>Status</strong></div><div>{escape(run['status'] or '')}</div>
      <div><strong>Form response UUID</strong></div><div><a href='/poll/quote-template/{escape(run['form_response_uuid'])}'>{escape(run['form_response_uuid'])}</a></div>
+      <div><strong>Job ID</strong></div><div>{job_id_html(run['job_uuid'])}</div>
      <div><strong>Job UUID</strong></div><div>{escape(run['job_uuid'] or '')}</div>
      <div><strong>Started</strong></div><div>{escape(run['started_at'] or '')}</div>
      <div><strong>Finished</strong></div><div>{escape(run['finished_at'] or '')}</div>
@@ -816,7 +878,7 @@ def list_polled_form_responses(q: str = Query(""), quote_only: int = Query(0), p
            f"<tr><td><a href='/poll/form-responses/{escape(row['uuid'])}'>{escape(row['uuid'])}</a></td>"
            f"<td>{escape(row['timestamp'] or '')}</td><td>{escape(row['edit_date'] or '')}</td>"
            f"<td>{escape(row['form_uuid'] or '')}<br>{quote_pill}</td>"
-            f"<td>{escape(row['regarding_object'] or '')}</td><td>{escape(row['regarding_object_uuid'] or '')}</td>"
+            f"<td>{escape(row['regarding_object'] or '')}</td><td>{job_id_html(row['regarding_object_uuid'])}</td><td>{escape(row['regarding_object_uuid'] or '')}</td>"
            f"<td>{escape(row['parse_status'] or '')}</td><td>{row['seen_count']}</td><td>{escape(row['last_seen_at'] or '')}</td></tr>"
        )

@@ -830,8 +892,8 @@ def list_polled_form_responses(q: str = Query(""), quote_only: int = Query(0), p
      </form>
    </div>
    <table>
-      <thead><tr><th>UUID</th><th>Timestamp</th><th>Edit date</th><th>Form UUID</th><th>Regarding</th><th>Object UUID</th><th>Parse</th><th>Seen</th><th>Last seen</th></tr></thead>
-      <tbody>{''.join(table_rows) or "<tr><td colspan='9'>No rows found.</td></tr>"}</tbody>
+      <thead><tr><th>UUID</th><th>Timestamp</th><th>Edit date</th><th>Form UUID</th><th>Regarding</th><th>Job ID</th><th>Object UUID</th><th>Parse</th><th>Seen</th><th>Last seen</th></tr></thead>
+      <tbody>{''.join(table_rows) or "<tr><td colspan='10'>No rows found.</td></tr>"}</tbody>
    </table>
    <div class='pagination'>
      {f"<a href='{link_with_params('/poll/form-responses', q=q, quote_only=quote_only, page=page-1)}'>← Prev</a>" if page > 1 else ''}
@@ -871,6 +933,7 @@ def polled_form_response_detail(form_response_uuid: str):
      <div><strong>Edit date</strong></div><div>{escape(row['edit_date'] or '')}</div>
      <div><strong>Form UUID</strong></div><div>{escape(row['form_uuid'] or '')}</div>
      <div><strong>Regarding object</strong></div><div>{escape(row['regarding_object'] or '')}</div>
+      <div><strong>Job ID</strong></div><div>{job_id_html(row['regarding_object_uuid'])}</div>
      <div><strong>Regarding UUID</strong></div><div>{escape(row['regarding_object_uuid'] or '')}</div>
      <div><strong>First seen</strong></div><div>{escape(row['first_seen_at'] or '')}</div>
      <div><strong>Last seen</strong></div><div>{escape(row['last_seen_at'] or '')}</div>
@@ -911,15 +974,15 @@ def list_polled_quote_templates(page: int = Query(1, ge=1)):
            material_count = "?"
        table_rows.append(
            f"<tr><td><a href='/poll/quote-template/{escape(row['form_response_uuid'])}'>{escape(row['form_response_uuid'])}</a></td>"
-            f"<td>{escape(row['discovered_at'] or '')}</td><td>{escape(row['job_uuid'] or '')}</td>"
+            f"<td>{escape(row['discovered_at'] or '')}</td><td>{job_id_html(row['job_uuid'])}</td><td>{escape(row['job_uuid'] or '')}</td>"
            f"<td>{escape(row['description'] or '')}</td><td>{material_count}</td>"
            f"<td>{escape(row['queued_at'] or '')}</td><td>{escape(row['process_status'] or '')}</td></tr>"
        )

    body = f"""
    <table>
-      <thead><tr><th>Form response UUID</th><th>Discovered</th><th>Job UUID</th><th>Description</th><th>Rows</th><th>Queued</th><th>Status</th></tr></thead>
-      <tbody>{''.join(table_rows) or "<tr><td colspan='7'>No quote template rows found.</td></tr>"}</tbody>
+      <thead><tr><th>Form response UUID</th><th>Discovered</th><th>Job ID</th><th>Job UUID</th><th>Description</th><th>Rows</th><th>Queued</th><th>Status</th></tr></thead>
+      <tbody>{''.join(table_rows) or "<tr><td colspan='8'>No quote template rows found.</td></tr>"}</tbody>
    </table>
    <div class='pagination'>
      {f"<a href='{link_with_params('/poll/quote-template', page=page-1)}'>← Prev</a>" if page > 1 else ''}
@@ -979,6 +1042,7 @@ def polled_quote_template_detail(form_response_uuid: str):
    <div class='card summary-grid'>
      <div><strong>Form response UUID</strong></div><div>{escape(row['form_response_uuid'])}</div>
      <div><strong>Discovered</strong></div><div>{escape(row['discovered_at'] or '')}</div>
+      <div><strong>Job ID</strong></div><div>{job_id_html(row['job_uuid'])}</div>
      <div><strong>Job UUID</strong></div><div>{escape(row['job_uuid'] or '')}</div>
      <div><strong>Form UUID</strong></div><div>{escape(row['form_uuid'] or '')}</div>
      <div><strong>Author</strong></div><div>{escape(row['author_name'] or '')}</div>