From 9dde6b28a171ef41567720bfb75b22680c56947c Mon Sep 17 00:00:00 2001 From: "Soren (Molty)" Date: Mon, 18 May 2026 11:54:57 +0930 Subject: [PATCH] I think this one was for the quote description field - but we got it wrong so we need to seatch and find the correct field...... --- apply_polled_quote_template_jobmaterials.py | 66 ++++++++++ poll_form_responses_since.py | 139 ++++++++++++++++++++ servicem8_inspector.py | 92 +++++++++++-- 3 files changed, 283 insertions(+), 14 deletions(-) diff --git a/apply_polled_quote_template_jobmaterials.py b/apply_polled_quote_template_jobmaterials.py index 845d71a..7b81015 100755 --- a/apply_polled_quote_template_jobmaterials.py +++ b/apply_polled_quote_template_jobmaterials.py @@ -134,6 +134,55 @@ def update_job_description(session: requests.Session, job_uuid: str, payload: di raise RuntimeError(f"Job description update failed: HTTP {response.status_code} :: {response.text[:1000]}") +def extract_company_name(job: Dict[str, Any]) -> str: + related = job.get("related") + if isinstance(related, dict): + company = related.get("company") + if isinstance(company, dict): + company_name = clean_text(company.get("name")) + if company_name: + return company_name + company = job.get("company") + if isinstance(company, dict): + company_name = clean_text(company.get("name")) + if company_name: + return company_name + return first_text(job.get("company_name"), job.get("customer_name")) + + +def upsert_job_metadata(conn: sqlite3.Connection, *, job_uuid: str, job: Dict[str, Any], source: str) -> None: + job_uuid = clean_text(job_uuid or job.get("uuid")) + if not job_uuid: + return + now = utc_now() + conn.execute( + """ + INSERT INTO job_metadata ( + job_uuid, generated_job_id, job_address, company_name, raw_json, + first_seen_at, last_seen_at, source + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(job_uuid) DO UPDATE SET + generated_job_id = excluded.generated_job_id, + job_address = excluded.job_address, + company_name = excluded.company_name, + raw_json = excluded.raw_json, + last_seen_at = excluded.last_seen_at, + source = excluded.source + """, + ( + job_uuid, + clean_text(job.get("generated_job_id")), + format_job_address(job), + extract_company_name(job), + json.dumps(job, ensure_ascii=False, sort_keys=True), + now, + now, + source, + ), + ) + conn.commit() + + def create_job_material(session: requests.Session, payload: dict) -> str: response = session.post(f"{BASE_URL}/jobmaterial.json", json=payload, timeout=REQUEST_TIMEOUT) if not response.ok: @@ -187,6 +236,21 @@ def get_conn(db_path: Path = POLL_DB_PATH) -> sqlite3.Connection: def init_apply_tables(conn: sqlite3.Connection) -> None: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS job_metadata ( + job_uuid TEXT PRIMARY KEY, + generated_job_id TEXT, + job_address TEXT, + company_name TEXT, + raw_json TEXT NOT NULL, + first_seen_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + source TEXT NOT NULL + ) + """ + ) + conn.execute("CREATE INDEX IF NOT EXISTS idx_job_metadata_generated_job_id ON job_metadata(generated_job_id)") conn.execute( """ CREATE TABLE IF NOT EXISTS quote_template_apply_runs ( @@ -467,6 +531,8 @@ def main() -> int: quote_description_source = clean_text(quote["description"]) job_details = retrieve_job(session, job_uuid) if quote_description_source else {} + if job_details: + upsert_job_metadata(conn, job_uuid=job_uuid, job=job_details, source=mode) job_update_payload = build_job_update_payload(quote_description_source, job_details) job_update_record_payload = { "endpoint": f"/job/{job_uuid}.json", diff --git a/poll_form_responses_since.py b/poll_form_responses_since.py index 955adb8..f336236 100755 --- a/poll_form_responses_since.py +++ b/poll_form_responses_since.py @@ -144,6 +144,21 @@ def init_db(db_path: Path) -> None: ) """ ) + conn.execute( + """ + CREATE TABLE IF NOT EXISTS job_metadata ( + job_uuid TEXT PRIMARY KEY, + generated_job_id TEXT, + job_address TEXT, + company_name TEXT, + raw_json TEXT NOT NULL, + first_seen_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + source TEXT NOT NULL + ) + """ + ) + conn.execute("CREATE INDEX IF NOT EXISTS idx_job_metadata_generated_job_id ON job_metadata(generated_job_id)") conn.commit() @@ -205,6 +220,109 @@ def fetch_form_responses( return response.status_code, data, filter_expr +def retrieve_job( + *, + api_key: str, + base_url: str, + job_uuid: str, + timeout: int, +) -> Dict[str, Any]: + response = requests.get( + f"{base_url.rstrip('/')}/job/{job_uuid}.json", + headers={"X-Api-Key": api_key, "Accept": "application/json"}, + timeout=timeout, + ) + if not response.ok: + raise RuntimeError(f"Job retrieve failed for {job_uuid}: HTTP {response.status_code}: {response.text[:1000]}") + data = response.json() + if not isinstance(data, dict): + raise RuntimeError(f"Job retrieve expected object response, got {type(data).__name__}") + return data + + +def clean_text(value: Any) -> str: + if value is None: + return "" + return str(value).replace("\r\n", "\n").replace("\r", "\n").strip() + + +def first_text(*values: Any) -> str: + for value in values: + text = clean_text(value) + if text: + return text + return "" + + +def format_job_address(job: Dict[str, Any]) -> str: + direct = first_text( + job.get("job_address"), + job.get("site_address"), + job.get("address"), + job.get("location_address"), + job.get("billing_address"), + ) + if direct: + return direct + + parts = [ + first_text(job.get("street"), job.get("street_address"), job.get("address_1"), job.get("address1")), + first_text(job.get("suburb"), job.get("city")), + first_text(job.get("state")), + first_text(job.get("postcode"), job.get("postal_code"), job.get("zip")), + ] + return " ".join(part for part in parts if part) + + +def extract_company_name(job: Dict[str, Any]) -> str: + related = job.get("related") + if isinstance(related, dict): + company = related.get("company") + if isinstance(company, dict): + company_name = clean_text(company.get("name")) + if company_name: + return company_name + company = job.get("company") + if isinstance(company, dict): + company_name = clean_text(company.get("name")) + if company_name: + return company_name + return first_text(job.get("company_name"), job.get("customer_name")) + + +def upsert_job_metadata(conn: sqlite3.Connection, *, job_uuid: str, job: Dict[str, Any], now: str, source: str) -> None: + job_uuid = clean_text(job_uuid or job.get("uuid")) + if not job_uuid: + return + + values = ( + job_uuid, + clean_text(job.get("generated_job_id")), + format_job_address(job), + extract_company_name(job), + json.dumps(job, ensure_ascii=False, sort_keys=True), + now, + now, + source, + ) + conn.execute( + """ + INSERT INTO job_metadata ( + job_uuid, generated_job_id, job_address, company_name, raw_json, + first_seen_at, last_seen_at, source + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(job_uuid) DO UPDATE SET + generated_job_id = excluded.generated_job_id, + job_address = excluded.job_address, + company_name = excluded.company_name, + raw_json = excluded.raw_json, + last_seen_at = excluded.last_seen_at, + source = excluded.source + """, + values, + ) + + def insert_or_update_raw( conn: sqlite3.Connection, row: Dict[str, Any], @@ -457,6 +575,7 @@ def main() -> int: inserted = updated = quote_matches = newly_queued = 0 now = utc_now() + fetched_job_uuids = set() if conn is not None: for row in rows: was_inserted, is_quote = insert_or_update_raw( @@ -469,6 +588,26 @@ def main() -> int: updated += 0 if was_inserted else 1 if is_quote: quote_matches += 1 + job_uuid = clean_text(row.get("regarding_object_uuid")) + if job_uuid and job_uuid not in fetched_job_uuids: + try: + job = retrieve_job( + api_key=api_key, + base_url=args.base_url, + job_uuid=job_uuid, + timeout=args.timeout, + ) + upsert_job_metadata(conn, job_uuid=job_uuid, job=job, now=now, source="formresponse_poll") + fetched_job_uuids.add(job_uuid) + except Exception as exc: + # Polling/parsing should still proceed if job metadata enrichment fails. + print( + json.dumps( + {"warning": "job_metadata_fetch_failed", "job_uuid": job_uuid, "error": str(exc)}, + ensure_ascii=False, + ), + file=sys.stderr, + ) if parse_and_store_quote_response( conn, row, diff --git a/servicem8_inspector.py b/servicem8_inspector.py index a248c7d..6a4f6e2 100644 --- a/servicem8_inspector.py +++ b/servicem8_inspector.py @@ -68,6 +68,7 @@ def html_page(title: str, body: str) -> HTMLResponse: code, pre { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; } pre { white-space: pre-wrap; word-break: break-word; background: #0f172a; color: #e2e8f0; padding: 14px; border-radius: 10px; overflow-x: auto; } .pill { display: inline-block; padding: 2px 8px; border-radius: 999px; background: #e0f2fe; color: #075985; font-size: 0.85rem; margin: 2px 4px 2px 0; } + .job-id { font-weight: 700; color: #111827; } .section { margin: 24px 0; } .toolbar { background: white; border: 1px solid #e5e7eb; border-radius: 10px; padding: 12px; margin-bottom: 16px; } input[type='text'] { padding: 8px; width: 280px; max-width: 100%; } @@ -139,6 +140,63 @@ def link_with_params(path, **params): return f"{path}?{urlencode(filtered)}" if filtered else path +def resolve_generated_job_id(job_uuid: str) -> str: + job_uuid = str(job_uuid or "").strip() + if not job_uuid: + return "" + + try: + with closing(get_poll_conn()) as conn: + row = conn.execute( + "select generated_job_id from job_metadata where job_uuid = ?", + (job_uuid,), + ).fetchone() + if row and row["generated_job_id"]: + return str(row["generated_job_id"]) + except sqlite3.Error: + pass + + try: + with closing(get_conn()) as conn: + row = conn.execute( + """ + with jobs as ( + select + json_extract(payload_json, '$.data.uuid') as job_uuid, + json_extract(payload_json, '$.data.generated_job_id') as generated_job_id, + received_at + from webhook_events + where json_extract(payload_json, '$.data.generated_job_id') is not null + union all + select + json_extract(payload_json, '$.related.job.uuid') as job_uuid, + json_extract(payload_json, '$.related.job.generated_job_id') as generated_job_id, + received_at + from webhook_form_responses + where json_extract(payload_json, '$.related.job.generated_job_id') is not null + ) + select generated_job_id + from jobs + where job_uuid = ? + and generated_job_id is not null + order by received_at desc + limit 1 + """, + (job_uuid,), + ).fetchone() + if row and row["generated_job_id"]: + return str(row["generated_job_id"]) + except sqlite3.Error: + pass + + return "" + + +def job_id_html(job_uuid: str) -> str: + generated_job_id = resolve_generated_job_id(job_uuid) + return f"{escape(generated_job_id)}" if generated_job_id else "" + + @app.get("/health") def health(): return {"ok": True, "db_path": DB_PATH, "state_db_path": STATE_DB_PATH, "poll_db_path": POLL_DB_PATH} @@ -459,6 +517,7 @@ def list_generated_materials(page: int = Query(1, ge=1)): table_rows.append( f"" f"{row['id']}" + f"{job_id_html(row['job_uuid'])}" f"{escape(row['job_uuid'] or '')}" f"{escape(row['form_response_uuid'] or '')}" f"{escape(row['job_material_uuid'] or '')}" @@ -470,8 +529,8 @@ def list_generated_materials(page: int = Query(1, ge=1)): body = f""" - - {''.join(table_rows) or ""} + + {''.join(table_rows) or ""}
IDJob UUIDForm response UUIDJob material UUIDKindSource questionUpdated
No rows found.
IDJob IDJob UUIDForm response UUIDJob material UUIDKindSource questionUpdated
No rows found.