From 868632bd211cdb1c9f0f08c42a638a3dbbee7800 Mon Sep 17 00:00:00 2001 From: "Soren (Molty)" Date: Mon, 4 May 2026 18:22:39 +1000 Subject: [PATCH] End of day updates, almost at completion of live cron setup with bash script wrapper now as ell --- PROJECT-PROGRESS.md | 205 ++++++++++++++------ apply_polled_quote_template_jobmaterials.py | 13 +- poll_and_apply_quote_templates.sh | 138 +++++++++++++ 3 files changed, 295 insertions(+), 61 deletions(-) create mode 100755 poll_and_apply_quote_templates.sh diff --git a/PROJECT-PROGRESS.md b/PROJECT-PROGRESS.md index b8573d2..b86482f 100644 --- a/PROJECT-PROGRESS.md +++ b/PROJECT-PROGRESS.md @@ -1,73 +1,158 @@ # ServiceM8 Project Progress -## Current Quote Template → JobMaterials Pipeline +## Current Direction — Soft Release Poller + Apply Pipeline -### Live receiver -- `servicem8_webhook_receiver.py` - - receives `form.response_created` - - stores the raw webhook payload in `servicem8_webhooks.db` - - checks incoming form responses for the Quote Template `form_uuid` - - if matched, parses and queues the derived jobMaterials payload to: - - `quote-template-jobmaterials-queue.jsonl` - - does **not** perform live ServiceM8 writes at this stage +We changed tune from relying mainly on ServiceM8 `form.response_created` webhooks to using a scheduled API poller as the operational safety-net/primary soft-release path. -### Parser -- `servicem8-quote-template-parser.py` - - parses Quote Template `field_data` - - extracts: - - description of works - - `Item 1..12` include lines - - `Works excluded 1..4` exclude lines - - extra descriptive include rows such as labour/scaffolding/equipment fields - - builds normalized `desired_job_materials` +Reason: ServiceM8 form webhooks are useful, but during testing some expected form completions did not reliably appear at the FastAPI listener. The ServiceM8 API endpoint `/api_1.0/formresponse.json` can be queried with a timestamp filter, so the safer operational pattern is now: -### Create/apply script -- `servicem8-create-jobmaterials-from-form-response.py` - - standalone script - - consumes a Quote Template form response JSON payload - - builds ServiceM8 Job Material API payloads - - runs in **dry-run by default** - - supports live create later with `--apply` - - records created/generated mappings into local state DB +1. poll recent form responses with `timestamp gt 'YYYY-MM-DD HH:MM:SS'` +2. store every returned API row locally +3. detect Quote Template form responses +4. parse them into desired jobMaterial rows +5. apply only previously unapplied Quote Template responses to ServiceM8 +6. track created jobMaterial UUIDs locally to avoid duplicate applies -### Local state tracking -- `servicem8_quote_materials_state.db` - - local SQLite DB for tracking generated jobMaterials - - intended fields include: - - job UUID - - form response UUID - - created job material UUID - - kind/source metadata +This keeps the live webhook receiver useful as capture/diagnostics, but no longer depends on webhook delivery for completeness. -### Queue/prepared output -- `quote-template-jobmaterials-queue.jsonl` - - lightweight queue/output file written by webhook stage - - contains parsed/prepared `desired_job_materials` objects - - no live update performed yet +## Active Soft-Release Flow -### Inspector -- `servicem8_inspector.py` - - read-only browser for webhook DB - - now also includes visibility of generated-materials state DB +### 1. Poll ServiceM8 form responses +- Script: `poll_form_responses_since.py` +- API endpoint: `/api_1.0/formresponse.json` +- Confirmed filter syntax: + - `?$filter=timestamp gt '2026-05-04 10:00:00'` +- Default operational DB: + - `servicem8_formresponse_poll.db` +- Poll queue/output: + - `quote-template-jobmaterials-poll-queue.jsonl` +- Quote Template form UUID: + - `3621b6be-1d19-4756-9ab4-9d5e4120f6d9` + +The poller stores all fetched form responses in `form_responses_raw`, then parses Quote Template matches into `quote_template_form_responses`. + +### 2. Parse Quote Template responses +- Parser: `servicem8_quote_template_parser.py` +- Extracts: + - description of works + - `Item 1..12` include lines + - `Works excluded 1..4` exclude lines + - extra descriptive include rows such as labour/materials/scaffolding/equipment fields +- Builds normalized `desired_job_materials` rows. + +### 3. Select/apply parsed Quote Template responses +- Script: `apply_polled_quote_template_jobmaterials.py` +- Safe behaviour built in: + - processes one `form_response_uuid` at a time when called directly + - dry-run by default + - `--apply` performs live ServiceM8 `jobmaterial.json` creates + - refuses duplicate apply when generated material rows already exist for that form response unless `--force` is used +- Apply tracking tables in `servicem8_formresponse_poll.db`: + - `quote_template_apply_runs` + - `quote_template_apply_run_rows` +- Created ServiceM8 job material mappings are recorded in: + - `servicem8_quote_materials_state.db` + +Current apply payload rules: +- All rows get: + - `tax_rate_uuid = 84e4dd28-06b3-452b-a796-1f58a20ac49b` + - `quantity = "0"` + - `price = ""` + - `displayed_amount = ""` + - `displayed_amount_is_tax_inclusive = ""` +- Header material UUIDs: + - `include_header` → `1924893b-917f-474a-adaa-2093bd622d4b` + - `exclude_header` → `4947bfd7-4875-48f7-9caf-2093b9751b9b` +- Non-header quote rows currently use: + - `f78b1d23-b9fa-40fe-a806-2425fe09cc0b` + +### 4. Wrapper for scheduled/operational use +- Script: `poll_and_apply_quote_templates.sh` +- Default behaviour: + - polls the last 24 hours from script start + - stores/parses results + - applies any parsed Quote Template responses that are not already marked/applied + - logs each run under `logs/poll-and-apply-YYYYMMDD-HHMMSS.log` +- Examples: + - `./poll_and_apply_quote_templates.sh` + - `./poll_and_apply_quote_templates.sh --hours 48` + - `./poll_and_apply_quote_templates.sh --since '2026-05-04 08:00:00'` + +This is the proposed scheduled entry point for soft release, e.g. every 10–30 minutes. + +## Live Webhook Receiver Status + +### Receiver +- Script: `servicem8_webhook_receiver.py` +- Still receives/stores: + - event webhooks + - object webhooks + - form-response webhooks +- DB: + - `servicem8_webhooks.db` +- Quote Template webhook queue: + - `quote-template-jobmaterials-queue.jsonl` + +Important: webhook handling still does **not** perform live ServiceM8 writes. Heavy work stays outside the FastAPI webhook request path. + +## Inspector / Web Viewer + +- Script: `servicem8_inspector.py` +- Current intended views include: + - webhook events + - webhook objects + - webhook form responses + - polled form responses + - parsed polled Quote Template responses + - poll runs + - dry-run/apply runs + - generated material state + +Note: the code was updated and tested on a temporary localhost port, but the existing live inspector process may need a manual/service restart before all new pages appear in the running viewer. + +## Required Files for Current Soft Release + +### Scripts +- `poll_and_apply_quote_templates.sh` — scheduled wrapper / main operational entry point +- `poll_form_responses_since.py` — polls ServiceM8 and populates poll DB +- `apply_polled_quote_template_jobmaterials.py` — applies parsed responses to ServiceM8 +- `servicem8_quote_template_parser.py` — parsing logic +- `servicem8_inspector.py` — web inspection/progress viewer +- `servicem8_webhook_receiver.py` — still useful for webhook capture/diagnostics + +### Databases / state +- `servicem8_formresponse_poll.db` — poll results, parsed quote responses, apply run status +- `servicem8_quote_materials_state.db` — created jobMaterial mapping/state to avoid duplicates +- `servicem8_webhooks.db` — webhook capture/archive/diagnostics + +### Queue/log files +- `quote-template-jobmaterials-poll-queue.jsonl` — poll-derived parsed queue/output +- `logs/poll-and-apply-*.log` — wrapper run logs +- `quote-template-jobmaterials-queue.jsonl` — older webhook-derived queue; still useful but not the primary soft-release path ## Current Status -Everything is staged and connected up to the point of: -- webhook receive -- UUID trigger check -- form parsing -- local queueing -- dry-run jobMaterial payload generation -- local state DB support -- inspector visibility -## Not Yet Enabled -- actual live POST creation of Job Materials into ServiceM8 during webhook processing -- any automatic update/delete reconciliation against live ServiceM8 records +Operational soft-release pieces are now in place: +- API polling confirmed +- Quote Template detection confirmed +- parsing confirmed +- dry-run/apply command path tested +- payload adjusted for current ServiceM8 requirements +- duplicate-apply guard in place +- wrapper script created for scheduled operation +- inspector updated for progress visibility + +## Not Yet Done / Next Steps + +- Restart/reload the live inspector process so the new poll/apply pages are available in the active web viewer. +- Decide schedule interval for `poll_and_apply_quote_templates.sh` — likely every 10 or 30 minutes. +- Run the wrapper manually for a soft-release smoke test with a controlled recent form response. +- After confidence builds, wire the wrapper into cron/system scheduling. +- Future hardening: add reconciliation/update/delete behaviour if ServiceM8 quote form responses are edited after initial apply. ## Design Notes -- Heavy lifting is intentionally kept **out of the live webhook handler**. -- The webhook handler is used only for: - - capture - - UUID gate - - parse/prepare/queue -- Live ServiceM8 mutation remains a separate step/script for safety. + +- Webhooks remain lightweight and non-mutating. +- Polling is now the reliable source of completeness. +- Applying to ServiceM8 is tracked locally and guarded against duplicates. +- The wrapper intentionally skips dry-run for soft release, but the underlying apply script still supports dry-run and duplicate protection. diff --git a/apply_polled_quote_template_jobmaterials.py b/apply_polled_quote_template_jobmaterials.py index 7a4d2d9..8e89c28 100755 --- a/apply_polled_quote_template_jobmaterials.py +++ b/apply_polled_quote_template_jobmaterials.py @@ -30,6 +30,8 @@ POLL_DB_PATH = Path(os.getenv("WEBHOOK_POLL_DB_PATH", SCRIPT_DIR / "servicem8_fo BASE_URL = os.getenv("SERVICEM8_BASE_URL", "https://api.servicem8.com/api_1.0") REQUEST_TIMEOUT = int(os.getenv("SERVICEM8_TIMEOUT", "30")) DEV_QUOTE_MATERIAL_UUID = "f78b1d23-b9fa-40fe-a806-2425fe09cc0b" +QUOTE_INCLUDE_HEADER_MATERIAL_UUID = "1924893b-917f-474a-adaa-2093bd622d4b" +QUOTE_EXCLUDE_HEADER_MATERIAL_UUID = "4947bfd7-4875-48f7-9caf-2093b9751b9b" DEV_QUOTE_TAX_RATE_UUID = "84e4dd28-06b3-452b-a796-1f58a20ac49b" @@ -37,10 +39,19 @@ def utc_now() -> str: return datetime.now(timezone.utc).isoformat() +def material_uuid_for_row(row: dict) -> str: + kind = row.get("kind", "") + if kind == "include_header": + return QUOTE_INCLUDE_HEADER_MATERIAL_UUID + if kind == "exclude_header": + return QUOTE_EXCLUDE_HEADER_MATERIAL_UUID + return DEV_QUOTE_MATERIAL_UUID + + def build_payload(job_uuid: str, row: dict) -> dict: return { "job_uuid": job_uuid, - "material_uuid": DEV_QUOTE_MATERIAL_UUID, + "material_uuid": material_uuid_for_row(row), "tax_rate_uuid": DEV_QUOTE_TAX_RATE_UUID, "name": row["name"], "quantity": "0", diff --git a/poll_and_apply_quote_templates.sh b/poll_and_apply_quote_templates.sh new file mode 100755 index 0000000..2f2eee7 --- /dev/null +++ b/poll_and_apply_quote_templates.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Poll ServiceM8 form responses, parse Quote Template responses, then apply any +# newly/unapplied parsed quote responses to ServiceM8 jobMaterials. +# +# Default window: last 24 hours from script start, in local system time. +# Override with: +# --since 'YYYY-MM-DD HH:MM:SS' +# --hours 48 +# +# This wrapper intentionally skips dry-run and calls --apply. The apply script +# still refuses duplicate applies unless --force is explicitly passed through. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +POLL_SCRIPT="$SCRIPT_DIR/poll_form_responses_since.py" +APPLY_SCRIPT="$SCRIPT_DIR/apply_polled_quote_template_jobmaterials.py" +DB_PATH="${WEBHOOK_POLL_DB_PATH:-$SCRIPT_DIR/servicem8_formresponse_poll.db}" +LOG_DIR="${WEBHOOK_RUN_LOG_DIR:-$SCRIPT_DIR/logs}" +QUOTE_TEMPLATE_FORM_UUID="${SERVICEM8_QUOTE_TEMPLATE_FORM_UUID:-3621b6be-1d19-4756-9ab4-9d5e4120f6d9}" + +SINCE="" +HOURS="24" +FORCE="0" + +usage() { + cat <&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ -z "$SINCE" ]]; then + SINCE="$(date -d "-${HOURS} hours" '+%Y-%m-%d %H:%M:%S')" +fi + +mkdir -p "$LOG_DIR" +RUN_ID="$(date '+%Y%m%d-%H%M%S')" +LOG_FILE="$LOG_DIR/poll-and-apply-$RUN_ID.log" + +exec > >(tee -a "$LOG_FILE") 2>&1 + +echo "== ServiceM8 Quote Template poll/apply run ==" +echo "Started: $(date --iso-8601=seconds)" +echo "Since: $SINCE" +echo "DB: $DB_PATH" +echo "Log: $LOG_FILE" +echo + +echo "== Polling form responses ==" +"$POLL_SCRIPT" --since "$SINCE" --summary-limit 20 + +echo +echo "== Finding unapplied parsed Quote Template responses ==" +mapfile -t FORM_RESPONSE_UUIDS < <( + python3 - "$DB_PATH" "$QUOTE_TEMPLATE_FORM_UUID" <<'PY' +import sqlite3 +import sys + +poll_db, quote_form_uuid = sys.argv[1], sys.argv[2] +conn = sqlite3.connect(poll_db) +conn.row_factory = sqlite3.Row +rows = conn.execute( + """ + select q.form_response_uuid + from quote_template_form_responses q + left join form_responses_raw r on r.uuid = q.form_response_uuid + where q.form_uuid = ? + and coalesce(q.process_status, '') != 'applied' + order by coalesce(r.timestamp, q.discovered_at) asc, q.form_response_uuid asc + """, + (quote_form_uuid,), +).fetchall() +for row in rows: + print(row["form_response_uuid"]) +PY +) + +if [[ ${#FORM_RESPONSE_UUIDS[@]} -eq 0 ]]; then + echo "No unapplied Quote Template responses found." + echo "Finished: $(date --iso-8601=seconds)" + exit 0 +fi + +printf 'Found %d unapplied Quote Template response(s):\n' "${#FORM_RESPONSE_UUIDS[@]}" +printf ' - %s\n' "${FORM_RESPONSE_UUIDS[@]}" + +echo +echo "== Applying to ServiceM8 ==" +APPLY_ARGS=(--apply --pretty) +if [[ "$FORCE" == "1" ]]; then + APPLY_ARGS+=(--force) +fi + +for uuid in "${FORM_RESPONSE_UUIDS[@]}"; do + echo + echo "-- Applying form_response_uuid=$uuid --" + "$APPLY_SCRIPT" --uuid "$uuid" "${APPLY_ARGS[@]}" +done + +echo +echo "Finished: $(date --iso-8601=seconds)" +echo "Log: $LOG_FILE"