diff --git a/servicem8_inspector.py b/servicem8_inspector.py index 0dc5214..d82f326 100644 --- a/servicem8_inspector.py +++ b/servicem8_inspector.py @@ -11,6 +11,7 @@ from fastapi.responses import HTMLResponse DB_PATH = os.getenv("WEBHOOK_DB_PATH", "./servicem8_webhooks.db") STATE_DB_PATH = os.getenv("WEBHOOK_STATE_DB_PATH", "./servicem8_quote_materials_state.db") +POLL_DB_PATH = os.getenv("WEBHOOK_POLL_DB_PATH", "./servicem8_formresponse_poll.db") APP_HOST = os.getenv("WEBHOOK_INSPECTOR_HOST", "0.0.0.0") APP_PORT = int(os.getenv("WEBHOOK_INSPECTOR_PORT", "18355")) PAGE_SIZE = 50 @@ -30,13 +31,21 @@ def get_state_conn(): return conn +def get_poll_conn(): + conn = sqlite3.connect(POLL_DB_PATH) + conn.row_factory = sqlite3.Row + return conn + + def html_page(title: str, body: str) -> HTMLResponse: nav = """ """ @@ -122,7 +131,7 @@ def link_with_params(path, **params): @app.get("/health") def health(): - return {"ok": True, "db_path": DB_PATH, "state_db_path": STATE_DB_PATH} + return {"ok": True, "db_path": DB_PATH, "state_db_path": STATE_DB_PATH, "poll_db_path": POLL_DB_PATH} @app.get("/", response_class=HTMLResponse) @@ -146,7 +155,27 @@ def dashboard(): except sqlite3.Error: pass + poll_counts = { + "poll_runs": 0, + "polled_form_responses": 0, + "polled_quote_templates": 0, + } + latest_poll_run = None + latest_polled_form = None + latest_polled_quote = None + try: + with closing(get_poll_conn()) as conn: + poll_counts["poll_runs"] = conn.execute("select count(*) from poll_runs").fetchone()[0] + poll_counts["polled_form_responses"] = conn.execute("select count(*) from form_responses_raw").fetchone()[0] + poll_counts["polled_quote_templates"] = conn.execute("select count(*) from quote_template_form_responses").fetchone()[0] + latest_poll_run = conn.execute("select finished_at from poll_runs order by id desc limit 1").fetchone() + latest_polled_form = conn.execute("select timestamp from form_responses_raw order by timestamp desc limit 1").fetchone() + latest_polled_quote = conn.execute("select discovered_at from quote_template_form_responses order by discovered_at desc limit 1").fetchone() + except sqlite3.Error: + pass + counts["generated_job_materials"] = state_count + counts.update(poll_counts) cards = "".join( f"
{escape(name)}
{count}
" @@ -158,10 +187,14 @@ def dashboard():
DB path
{escape(DB_PATH)}
State DB path
{escape(STATE_DB_PATH)}
+
Poll DB path
{escape(POLL_DB_PATH)}
Latest event
{escape(latest_event[0] if latest_event else '—')}
Latest object
{escape(latest_object[0] if latest_object else '—')}
Latest form response
{escape(latest_form[0] if latest_form else '—')}
Latest generated material row
{escape(latest_generated[0] if latest_generated else '—')}
+
Latest poll run
{escape(latest_poll_run[0] if latest_poll_run else '—')}
+
Latest polled form timestamp
{escape(latest_polled_form[0] if latest_polled_form else '—')}
+
Latest polled quote discovered
{escape(latest_polled_quote[0] if latest_polled_quote else '—')}
@@ -169,7 +202,10 @@ def dashboard():
@@ -504,6 +540,235 @@ def form_response_detail(row_id: int): return html_page(f"Form response {row_id}", body) +@app.get("/poll/runs", response_class=HTMLResponse) +def list_poll_runs(page: int = Query(1, ge=1)): + offset = (page - 1) * PAGE_SIZE + try: + with closing(get_poll_conn()) as conn: + rows = conn.execute( + """ + select id, started_at, finished_at, since_value, filter_field, + fetched_count, inserted_count, updated_count, + quote_match_count, newly_queued_count, error + from poll_runs order by id desc limit ? offset ? + """, + (PAGE_SIZE, offset), + ).fetchall() + except sqlite3.Error as e: + return html_page("Poll runs", f"
Poll DB unavailable: {escape(str(e))}
") + + table_rows = [] + for row in rows: + table_rows.append( + f"{row['id']}{escape(row['started_at'] or '')}" + f"{escape(row['finished_at'] or '')}" + f"{escape(row['filter_field'] or '')} gt {escape(row['since_value'] or '')}" + f"{row['fetched_count']}{row['inserted_count']}{row['updated_count']}" + f"{row['quote_match_count']}{row['newly_queued_count']}" + f"{escape((row['error'] or '')[:180])}" + ) + + body = f""" + + + {''.join(table_rows) or ""} +
IDStartedFinishedFilterFetchedInsertedUpdatedQuote matchesQueuedError
No poll runs found.
+ + """ + return html_page("Poll runs", body) + + +@app.get("/poll/form-responses", response_class=HTMLResponse) +def list_polled_form_responses(q: str = Query(""), quote_only: int = Query(0), page: int = Query(1, ge=1)): + offset = (page - 1) * PAGE_SIZE + clauses = [] + params = [] + if quote_only: + clauses.append("is_quote_template = 1") + if q.strip(): + like = f"%{q.strip()}%" + clauses.append("(uuid like ? or form_uuid like ? or regarding_object_uuid like ? or timestamp like ? or edit_date like ? or raw_json like ?)") + params.extend([like, like, like, like, like, like]) + where = " where " + " and ".join(clauses) if clauses else "" + + try: + with closing(get_poll_conn()) as conn: + rows = conn.execute( + f""" + select uuid, first_seen_at, last_seen_at, seen_count, timestamp, edit_date, + form_uuid, regarding_object, regarding_object_uuid, + is_quote_template, parse_status, parse_error + from form_responses_raw {where} + order by timestamp desc, edit_date desc limit ? offset ? + """, + (*params, PAGE_SIZE, offset), + ).fetchall() + except sqlite3.Error as e: + return html_page("Polled form responses", f"
Poll DB unavailable: {escape(str(e))}
") + + table_rows = [] + for row in rows: + quote_pill = "Quote Template" if row["is_quote_template"] else "" + table_rows.append( + f"{escape(row['uuid'])}" + f"{escape(row['timestamp'] or '')}{escape(row['edit_date'] or '')}" + f"{escape(row['form_uuid'] or '')}
{quote_pill}" + f"{escape(row['regarding_object'] or '')}{escape(row['regarding_object_uuid'] or '')}" + f"{escape(row['parse_status'] or '')}{row['seen_count']}{escape(row['last_seen_at'] or '')}" + ) + + body = f""" +
+
+ + + + +
+
+ + + {''.join(table_rows) or ""} +
UUIDTimestampEdit dateForm UUIDRegardingObject UUIDParseSeenLast seen
No rows found.
+ + """ + return html_page("Polled form responses", body) + + +@app.get("/poll/form-responses/{form_response_uuid}", response_class=HTMLResponse) +def polled_form_response_detail(form_response_uuid: str): + try: + with closing(get_poll_conn()) as conn: + row = conn.execute("select * from form_responses_raw where uuid = ?", (form_response_uuid,)).fetchone() + quote = conn.execute("select * from quote_template_form_responses where form_response_uuid = ?", (form_response_uuid,)).fetchone() + except sqlite3.Error as e: + return html_page("Polled form response", f"
Poll DB unavailable: {escape(str(e))}
") + if not row: + raise HTTPException(status_code=404, detail="Polled form response not found") + + raw = parse_json_field(row["raw_json"], {}) or {} + field_data = parse_json_field(raw.get("field_data"), []) or [] + field_rows = [] + if isinstance(field_data, list): + for item in sorted(field_data, key=lambda x: x.get("SortOrder", 0) if isinstance(x, dict) else 0): + if isinstance(item, dict): + field_rows.append( + f"{escape(str(item.get('SortOrder', '')))}{escape(item.get('FieldType', ''))}" + f"{escape(item.get('Question', ''))}{escape(item.get('Response', ''))}" + ) + quote_link = f"
Parsed quote
Open parsed Quote Template view
" if quote else "" + + body = f""" +
+
UUID
{escape(row['uuid'])}
+
Timestamp
{escape(row['timestamp'] or '')}
+
Edit date
{escape(row['edit_date'] or '')}
+
Form UUID
{escape(row['form_uuid'] or '')}
+
Regarding object
{escape(row['regarding_object'] or '')}
+
Regarding UUID
{escape(row['regarding_object_uuid'] or '')}
+
First seen
{escape(row['first_seen_at'] or '')}
+
Last seen
{escape(row['last_seen_at'] or '')}
+
Seen count
{row['seen_count']}
+
Parse status
{escape(row['parse_status'] or '')}
+
Parse error
{escape(row['parse_error'] or '')}
+ {quote_link} +
+

Decoded field data

{''.join(field_rows) or ""}
OrderTypeQuestionResponse
No decoded field data.
+

Raw API row

{escape(pretty_json(raw))}
+ """ + return html_page(f"Polled form response {form_response_uuid}", body) + + +@app.get("/poll/quote-template", response_class=HTMLResponse) +def list_polled_quote_templates(page: int = Query(1, ge=1)): + offset = (page - 1) * PAGE_SIZE + try: + with closing(get_poll_conn()) as conn: + rows = conn.execute( + """ + select form_response_uuid, discovered_at, job_uuid, form_uuid, author_name, + description, desired_job_materials_json, queued_at, processed_at, + process_status, process_error + from quote_template_form_responses + order by discovered_at desc, form_response_uuid desc limit ? offset ? + """, + (PAGE_SIZE, offset), + ).fetchall() + except sqlite3.Error as e: + return html_page("Polled Quote Template responses", f"
Poll DB unavailable: {escape(str(e))}
") + + table_rows = [] + for row in rows: + try: + material_count = len(json.loads(row["desired_job_materials_json"] or "[]")) + except Exception: + material_count = "?" + table_rows.append( + f"{escape(row['form_response_uuid'])}" + f"{escape(row['discovered_at'] or '')}{escape(row['job_uuid'] or '')}" + f"{escape(row['description'] or '')}{material_count}" + f"{escape(row['queued_at'] or '')}{escape(row['process_status'] or '')}" + ) + + body = f""" + + + {''.join(table_rows) or ""} +
Form response UUIDDiscoveredJob UUIDDescriptionRowsQueuedStatus
No quote template rows found.
+ + """ + return html_page("Polled Quote Template responses", body) + + +@app.get("/poll/quote-template/{form_response_uuid}", response_class=HTMLResponse) +def polled_quote_template_detail(form_response_uuid: str): + try: + with closing(get_poll_conn()) as conn: + row = conn.execute("select * from quote_template_form_responses where form_response_uuid = ?", (form_response_uuid,)).fetchone() + except sqlite3.Error as e: + return html_page("Polled Quote Template response", f"
Poll DB unavailable: {escape(str(e))}
") + if not row: + raise HTTPException(status_code=404, detail="Polled quote template response not found") + + parsed = parse_json_field(row["parsed_json"], {}) or {} + materials = parse_json_field(row["desired_job_materials_json"], []) or [] + material_rows = [] + for item in materials: + material_rows.append( + f"{escape(str(item.get('sort_order', '')))}{escape(item.get('kind', ''))}" + f"{escape(item.get('name', ''))}{escape(item.get('material_uuid', ''))}" + f"{escape(item.get('source_question', ''))}" + ) + + body = f""" +
+
Form response UUID
{escape(row['form_response_uuid'])}
+
Discovered
{escape(row['discovered_at'] or '')}
+
Job UUID
{escape(row['job_uuid'] or '')}
+
Form UUID
{escape(row['form_uuid'] or '')}
+
Author
{escape(row['author_name'] or '')}
+
Description
{escape(row['description'] or '')}
+
Queued
{escape(row['queued_at'] or '')}
+
Processed
{escape(row['processed_at'] or '')}
+
Status
{escape(row['process_status'] or '')}
+
Error
{escape(row['process_error'] or '')}
+
Raw polled response
Open raw polled form response
+
+

Desired jobMaterial rows

{''.join(material_rows) or ""}
SortKindNameMaterial UUIDSource question
No desired jobMaterial rows.
+

Parsed JSON

{escape(pretty_json(parsed))}
+ """ + return html_page(f"Polled Quote Template {form_response_uuid}", body) + + if __name__ == "__main__": import uvicorn