Adding updated inspector.py for human viewable formresponse.json requests

servicem8_inspector.py

@@ -11,6 +11,7 @@ from fastapi.responses import HTMLResponse
 
 DB_PATH = os.getenv("WEBHOOK_DB_PATH", "./servicem8_webhooks.db")
 STATE_DB_PATH = os.getenv("WEBHOOK_STATE_DB_PATH", "./servicem8_quote_materials_state.db")
+POLL_DB_PATH = os.getenv("WEBHOOK_POLL_DB_PATH", "./servicem8_formresponse_poll.db")
 APP_HOST = os.getenv("WEBHOOK_INSPECTOR_HOST", "0.0.0.0")
 APP_PORT = int(os.getenv("WEBHOOK_INSPECTOR_PORT", "18355"))
 PAGE_SIZE = 50
@@ -30,13 +31,21 @@ def get_state_conn():
     return conn
 
 
+def get_poll_conn():
+    conn = sqlite3.connect(POLL_DB_PATH)
+    conn.row_factory = sqlite3.Row
+    return conn
+
+
 def html_page(title: str, body: str) -> HTMLResponse:
     nav = """
     <nav>
       <a href='/'>Dashboard</a>
       <a href='/events'>Events</a>
       <a href='/objects'>Objects</a>
-      <a href='/form-responses'>Form responses</a>
+      <a href='/form-responses'>Webhook form responses</a>
+      <a href='/poll/form-responses'>Polled form responses</a>
+      <a href='/poll/quote-template'>Polled quote templates</a>
       <a href='/generated-materials'>Generated materials</a>
     </nav>
     """
@@ -122,7 +131,7 @@ def link_with_params(path, **params):
 
 @app.get("/health")
 def health():
-    return {"ok": True, "db_path": DB_PATH, "state_db_path": STATE_DB_PATH}
+    return {"ok": True, "db_path": DB_PATH, "state_db_path": STATE_DB_PATH, "poll_db_path": POLL_DB_PATH}
 
 
 @app.get("/", response_class=HTMLResponse)
@@ -146,7 +155,27 @@ def dashboard():
     except sqlite3.Error:
         pass
 
+    poll_counts = {
+        "poll_runs": 0,
+        "polled_form_responses": 0,
+        "polled_quote_templates": 0,
+    }
+    latest_poll_run = None
+    latest_polled_form = None
+    latest_polled_quote = None
+    try:
+        with closing(get_poll_conn()) as conn:
+            poll_counts["poll_runs"] = conn.execute("select count(*) from poll_runs").fetchone()[0]
+            poll_counts["polled_form_responses"] = conn.execute("select count(*) from form_responses_raw").fetchone()[0]
+            poll_counts["polled_quote_templates"] = conn.execute("select count(*) from quote_template_form_responses").fetchone()[0]
+            latest_poll_run = conn.execute("select finished_at from poll_runs order by id desc limit 1").fetchone()
+            latest_polled_form = conn.execute("select timestamp from form_responses_raw order by timestamp desc limit 1").fetchone()
+            latest_polled_quote = conn.execute("select discovered_at from quote_template_form_responses order by discovered_at desc limit 1").fetchone()
+    except sqlite3.Error:
+        pass
+
     counts["generated_job_materials"] = state_count
+    counts.update(poll_counts)
 
     cards = "".join(
         f"<div class='card'><div class='muted'>{escape(name)}</div><div class='big'>{count}</div></div>"
@@ -158,10 +187,14 @@ def dashboard():
       <div class='summary-grid'>
         <div><strong>DB path</strong></div><div><code>{escape(DB_PATH)}</code></div>
         <div><strong>State DB path</strong></div><div><code>{escape(STATE_DB_PATH)}</code></div>
+        <div><strong>Poll DB path</strong></div><div><code>{escape(POLL_DB_PATH)}</code></div>
         <div><strong>Latest event</strong></div><div>{escape(latest_event[0] if latest_event else '—')}</div>
         <div><strong>Latest object</strong></div><div>{escape(latest_object[0] if latest_object else '—')}</div>
         <div><strong>Latest form response</strong></div><div>{escape(latest_form[0] if latest_form else '—')}</div>
         <div><strong>Latest generated material row</strong></div><div>{escape(latest_generated[0] if latest_generated else '—')}</div>
+        <div><strong>Latest poll run</strong></div><div>{escape(latest_poll_run[0] if latest_poll_run else '—')}</div>
+        <div><strong>Latest polled form timestamp</strong></div><div>{escape(latest_polled_form[0] if latest_polled_form else '—')}</div>
+        <div><strong>Latest polled quote discovered</strong></div><div>{escape(latest_polled_quote[0] if latest_polled_quote else '—')}</div>
       </div>
     </div>
     <div class='section'>
@@ -169,7 +202,10 @@ def dashboard():
       <ul>
         <li><a href='/events'>Browse webhook events</a></li>
         <li><a href='/objects'>Browse object webhooks</a></li>
-        <li><a href='/form-responses'>Browse form responses</a></li>
+        <li><a href='/form-responses'>Browse webhook form responses</a></li>
+        <li><a href='/poll/form-responses'>Browse polled form responses</a></li>
+        <li><a href='/poll/quote-template'>Browse parsed polled Quote Template responses</a></li>
+        <li><a href='/poll/runs'>Browse poll runs</a></li>
         <li><a href='/generated-materials'>Browse generated job-material state</a></li>
       </ul>
     </div>
@@ -504,6 +540,235 @@ def form_response_detail(row_id: int):
     return html_page(f"Form response {row_id}", body)
 
 
+@app.get("/poll/runs", response_class=HTMLResponse)
+def list_poll_runs(page: int = Query(1, ge=1)):
+    offset = (page - 1) * PAGE_SIZE
+    try:
+        with closing(get_poll_conn()) as conn:
+            rows = conn.execute(
+                """
+                select id, started_at, finished_at, since_value, filter_field,
+                       fetched_count, inserted_count, updated_count,
+                       quote_match_count, newly_queued_count, error
+                from poll_runs order by id desc limit ? offset ?
+                """,
+                (PAGE_SIZE, offset),
+            ).fetchall()
+    except sqlite3.Error as e:
+        return html_page("Poll runs", f"<div class='card'>Poll DB unavailable: {escape(str(e))}</div>")
+
+    table_rows = []
+    for row in rows:
+        table_rows.append(
+            f"<tr><td>{row['id']}</td><td>{escape(row['started_at'] or '')}</td>"
+            f"<td>{escape(row['finished_at'] or '')}</td>"
+            f"<td>{escape(row['filter_field'] or '')} gt {escape(row['since_value'] or '')}</td>"
+            f"<td>{row['fetched_count']}</td><td>{row['inserted_count']}</td><td>{row['updated_count']}</td>"
+            f"<td>{row['quote_match_count']}</td><td>{row['newly_queued_count']}</td>"
+            f"<td>{escape((row['error'] or '')[:180])}</td></tr>"
+        )
+
+    body = f"""
+    <table>
+      <thead><tr><th>ID</th><th>Started</th><th>Finished</th><th>Filter</th><th>Fetched</th><th>Inserted</th><th>Updated</th><th>Quote matches</th><th>Queued</th><th>Error</th></tr></thead>
+      <tbody>{''.join(table_rows) or "<tr><td colspan='10'>No poll runs found.</td></tr>"}</tbody>
+    </table>
+    <div class='pagination'>
+      {f"<a href='{link_with_params('/poll/runs', page=page-1)}'>← Prev</a>" if page > 1 else ''}
+      <a href='{link_with_params('/poll/runs', page=page+1)}'>Next →</a>
+    </div>
+    """
+    return html_page("Poll runs", body)
+
+
+@app.get("/poll/form-responses", response_class=HTMLResponse)
+def list_polled_form_responses(q: str = Query(""), quote_only: int = Query(0), page: int = Query(1, ge=1)):
+    offset = (page - 1) * PAGE_SIZE
+    clauses = []
+    params = []
+    if quote_only:
+        clauses.append("is_quote_template = 1")
+    if q.strip():
+        like = f"%{q.strip()}%"
+        clauses.append("(uuid like ? or form_uuid like ? or regarding_object_uuid like ? or timestamp like ? or edit_date like ? or raw_json like ?)")
+        params.extend([like, like, like, like, like, like])
+    where = " where " + " and ".join(clauses) if clauses else ""
+
+    try:
+        with closing(get_poll_conn()) as conn:
+            rows = conn.execute(
+                f"""
+                select uuid, first_seen_at, last_seen_at, seen_count, timestamp, edit_date,
+                       form_uuid, regarding_object, regarding_object_uuid,
+                       is_quote_template, parse_status, parse_error
+                from form_responses_raw {where}
+                order by timestamp desc, edit_date desc limit ? offset ?
+                """,
+                (*params, PAGE_SIZE, offset),
+            ).fetchall()
+    except sqlite3.Error as e:
+        return html_page("Polled form responses", f"<div class='card'>Poll DB unavailable: {escape(str(e))}</div>")
+
+    table_rows = []
+    for row in rows:
+        quote_pill = "<span class='pill'>Quote Template</span>" if row["is_quote_template"] else ""
+        table_rows.append(
+            f"<tr><td><a href='/poll/form-responses/{escape(row['uuid'])}'>{escape(row['uuid'])}</a></td>"
+            f"<td>{escape(row['timestamp'] or '')}</td><td>{escape(row['edit_date'] or '')}</td>"
+            f"<td>{escape(row['form_uuid'] or '')}<br>{quote_pill}</td>"
+            f"<td>{escape(row['regarding_object'] or '')}</td><td>{escape(row['regarding_object_uuid'] or '')}</td>"
+            f"<td>{escape(row['parse_status'] or '')}</td><td>{row['seen_count']}</td><td>{escape(row['last_seen_at'] or '')}</td></tr>"
+        )
+
+    body = f"""
+    <div class='toolbar'>
+      <form method='get'>
+        <label>Search <input type='text' name='q' value='{escape(q)}' placeholder='uuid, job uuid, form uuid, timestamp'></label>
+        <label><input type='checkbox' name='quote_only' value='1' {'checked' if quote_only else ''}> Quote Template only</label>
+        <input type='hidden' name='page' value='1'>
+        <button type='submit'>Filter</button>
+      </form>
+    </div>
+    <table>
+      <thead><tr><th>UUID</th><th>Timestamp</th><th>Edit date</th><th>Form UUID</th><th>Regarding</th><th>Object UUID</th><th>Parse</th><th>Seen</th><th>Last seen</th></tr></thead>
+      <tbody>{''.join(table_rows) or "<tr><td colspan='9'>No rows found.</td></tr>"}</tbody>
+    </table>
+    <div class='pagination'>
+      {f"<a href='{link_with_params('/poll/form-responses', q=q, quote_only=quote_only, page=page-1)}'>← Prev</a>" if page > 1 else ''}
+      <a href='{link_with_params('/poll/form-responses', q=q, quote_only=quote_only, page=page+1)}'>Next →</a>
+    </div>
+    """
+    return html_page("Polled form responses", body)
+
+
+@app.get("/poll/form-responses/{form_response_uuid}", response_class=HTMLResponse)
+def polled_form_response_detail(form_response_uuid: str):
+    try:
+        with closing(get_poll_conn()) as conn:
+            row = conn.execute("select * from form_responses_raw where uuid = ?", (form_response_uuid,)).fetchone()
+            quote = conn.execute("select * from quote_template_form_responses where form_response_uuid = ?", (form_response_uuid,)).fetchone()
+    except sqlite3.Error as e:
+        return html_page("Polled form response", f"<div class='card'>Poll DB unavailable: {escape(str(e))}</div>")
+    if not row:
+        raise HTTPException(status_code=404, detail="Polled form response not found")
+
+    raw = parse_json_field(row["raw_json"], {}) or {}
+    field_data = parse_json_field(raw.get("field_data"), []) or []
+    field_rows = []
+    if isinstance(field_data, list):
+        for item in sorted(field_data, key=lambda x: x.get("SortOrder", 0) if isinstance(x, dict) else 0):
+            if isinstance(item, dict):
+                field_rows.append(
+                    f"<tr><td>{escape(str(item.get('SortOrder', '')))}</td><td>{escape(item.get('FieldType', ''))}</td>"
+                    f"<td>{escape(item.get('Question', ''))}</td><td>{escape(item.get('Response', ''))}</td></tr>"
+                )
+    quote_link = f"<div><strong>Parsed quote</strong></div><div><a href='/poll/quote-template/{escape(form_response_uuid)}'>Open parsed Quote Template view</a></div>" if quote else ""
+
+    body = f"""
+    <div class='card summary-grid'>
+      <div><strong>UUID</strong></div><div>{escape(row['uuid'])}</div>
+      <div><strong>Timestamp</strong></div><div>{escape(row['timestamp'] or '')}</div>
+      <div><strong>Edit date</strong></div><div>{escape(row['edit_date'] or '')}</div>
+      <div><strong>Form UUID</strong></div><div>{escape(row['form_uuid'] or '')}</div>
+      <div><strong>Regarding object</strong></div><div>{escape(row['regarding_object'] or '')}</div>
+      <div><strong>Regarding UUID</strong></div><div>{escape(row['regarding_object_uuid'] or '')}</div>
+      <div><strong>First seen</strong></div><div>{escape(row['first_seen_at'] or '')}</div>
+      <div><strong>Last seen</strong></div><div>{escape(row['last_seen_at'] or '')}</div>
+      <div><strong>Seen count</strong></div><div>{row['seen_count']}</div>
+      <div><strong>Parse status</strong></div><div>{escape(row['parse_status'] or '')}</div>
+      <div><strong>Parse error</strong></div><div>{escape(row['parse_error'] or '')}</div>
+      {quote_link}
+    </div>
+    <div class='section'><h2>Decoded field data</h2><table><thead><tr><th>Order</th><th>Type</th><th>Question</th><th>Response</th></tr></thead><tbody>{''.join(field_rows) or "<tr><td colspan='4'>No decoded field data.</td></tr>"}</tbody></table></div>
+    <div class='section'><h2>Raw API row</h2><pre>{escape(pretty_json(raw))}</pre></div>
+    """
+    return html_page(f"Polled form response {form_response_uuid}", body)
+
+
+@app.get("/poll/quote-template", response_class=HTMLResponse)
+def list_polled_quote_templates(page: int = Query(1, ge=1)):
+    offset = (page - 1) * PAGE_SIZE
+    try:
+        with closing(get_poll_conn()) as conn:
+            rows = conn.execute(
+                """
+                select form_response_uuid, discovered_at, job_uuid, form_uuid, author_name,
+                       description, desired_job_materials_json, queued_at, processed_at,
+                       process_status, process_error
+                from quote_template_form_responses
+                order by discovered_at desc, form_response_uuid desc limit ? offset ?
+                """,
+                (PAGE_SIZE, offset),
+            ).fetchall()
+    except sqlite3.Error as e:
+        return html_page("Polled Quote Template responses", f"<div class='card'>Poll DB unavailable: {escape(str(e))}</div>")
+
+    table_rows = []
+    for row in rows:
+        try:
+            material_count = len(json.loads(row["desired_job_materials_json"] or "[]"))
+        except Exception:
+            material_count = "?"
+        table_rows.append(
+            f"<tr><td><a href='/poll/quote-template/{escape(row['form_response_uuid'])}'>{escape(row['form_response_uuid'])}</a></td>"
+            f"<td>{escape(row['discovered_at'] or '')}</td><td>{escape(row['job_uuid'] or '')}</td>"
+            f"<td>{escape(row['description'] or '')}</td><td>{material_count}</td>"
+            f"<td>{escape(row['queued_at'] or '')}</td><td>{escape(row['process_status'] or '')}</td></tr>"
+        )
+
+    body = f"""
+    <table>
+      <thead><tr><th>Form response UUID</th><th>Discovered</th><th>Job UUID</th><th>Description</th><th>Rows</th><th>Queued</th><th>Status</th></tr></thead>
+      <tbody>{''.join(table_rows) or "<tr><td colspan='7'>No quote template rows found.</td></tr>"}</tbody>
+    </table>
+    <div class='pagination'>
+      {f"<a href='{link_with_params('/poll/quote-template', page=page-1)}'>← Prev</a>" if page > 1 else ''}
+      <a href='{link_with_params('/poll/quote-template', page=page+1)}'>Next →</a>
+    </div>
+    """
+    return html_page("Polled Quote Template responses", body)
+
+
+@app.get("/poll/quote-template/{form_response_uuid}", response_class=HTMLResponse)
+def polled_quote_template_detail(form_response_uuid: str):
+    try:
+        with closing(get_poll_conn()) as conn:
+            row = conn.execute("select * from quote_template_form_responses where form_response_uuid = ?", (form_response_uuid,)).fetchone()
+    except sqlite3.Error as e:
+        return html_page("Polled Quote Template response", f"<div class='card'>Poll DB unavailable: {escape(str(e))}</div>")
+    if not row:
+        raise HTTPException(status_code=404, detail="Polled quote template response not found")
+
+    parsed = parse_json_field(row["parsed_json"], {}) or {}
+    materials = parse_json_field(row["desired_job_materials_json"], []) or []
+    material_rows = []
+    for item in materials:
+        material_rows.append(
+            f"<tr><td>{escape(str(item.get('sort_order', '')))}</td><td>{escape(item.get('kind', ''))}</td>"
+            f"<td>{escape(item.get('name', ''))}</td><td>{escape(item.get('material_uuid', ''))}</td>"
+            f"<td>{escape(item.get('source_question', ''))}</td></tr>"
+        )
+
+    body = f"""
+    <div class='card summary-grid'>
+      <div><strong>Form response UUID</strong></div><div>{escape(row['form_response_uuid'])}</div>
+      <div><strong>Discovered</strong></div><div>{escape(row['discovered_at'] or '')}</div>
+      <div><strong>Job UUID</strong></div><div>{escape(row['job_uuid'] or '')}</div>
+      <div><strong>Form UUID</strong></div><div>{escape(row['form_uuid'] or '')}</div>
+      <div><strong>Author</strong></div><div>{escape(row['author_name'] or '')}</div>
+      <div><strong>Description</strong></div><div>{escape(row['description'] or '')}</div>
+      <div><strong>Queued</strong></div><div>{escape(row['queued_at'] or '')}</div>
+      <div><strong>Processed</strong></div><div>{escape(row['processed_at'] or '')}</div>
+      <div><strong>Status</strong></div><div>{escape(row['process_status'] or '')}</div>
+      <div><strong>Error</strong></div><div>{escape(row['process_error'] or '')}</div>
+      <div><strong>Raw polled response</strong></div><div><a href='/poll/form-responses/{escape(row['form_response_uuid'])}'>Open raw polled form response</a></div>
+    </div>
+    <div class='section'><h2>Desired jobMaterial rows</h2><table><thead><tr><th>Sort</th><th>Kind</th><th>Name</th><th>Material UUID</th><th>Source question</th></tr></thead><tbody>{''.join(material_rows) or "<tr><td colspan='5'>No desired jobMaterial rows.</td></tr>"}</tbody></table></div>
+    <div class='section'><h2>Parsed JSON</h2><pre>{escape(pretty_json(parsed))}</pre></div>
+    """
+    return html_page(f"Polled Quote Template {form_response_uuid}", body)
+
+
 if __name__ == "__main__":
     import uvicorn