Raahat Scenario Harness

Curated conversation replay against the real runtime with stepwise diagnostics, readable deltas, and deterministic checks.

Ops Console Simulator

Clinic Recommendation Ask

concierge_clinic_recommendation · concierge · 2/2 turns executed

Reset Step Next Run Full

Scenario Summary

IDconcierge_clinic_recommendation
TitleClinic Recommendation Ask
Categoryconcierge
Turns2/2
Run Statecompleted
clinic recommendation multi_turn

Seed State

UserUnknown user
Phone919900000304
Seed Workflow—
Seed State Machine—

Current Runtime State

Routemedical_triage
Workflowmedical_triage
State Machinemedical_triage
Stage—

Checks And Lifecycle

Expectations Passed0
Expectations Failed2
Heuristic Warnings0
Medical supersession
Final assessmentLikely follow-up needed: at least one explicit check failed during this run.

Final Outcome

Completed

Completed vs partial run state.

Final Route

medical_triage

Current route at the end of the replay.

Final Workflow / State

medical_triage

Workflow preferred; state machine shown if no workflow is active.

Final Stage

—

Resolved from workflow progress or active snapshot.

Expectation Totals

Pass 0 Fail 2

Heuristic Totals

Warnings 0

Missing Fields

No unresolved workflow fields.

Lifecycle Events Seen

Medical supersession
Turn 1
medical_triage medical_triage intent_match
Expectation failure
LIKELY BUG PASS
User
Can you suggest a clinic nearby in Baner
Assistant
—

What Changed This Turn

  • No material state change.

Expectation And Heuristic Results

Expectations
LIKELY BUG: workflow_name expected 'concierge' but got 'medical_triage'
Heuristics
PASS: Medical turn with location had clinic-aware handling
Routing And Interpretation 
{
  "confidence": 0.9,
  "reason_codes": [
    "intent_match"
  ],
  "route_type": "medical_triage",
  "workflow_name": "medical_triage"
}
{
  "confidence": 0.9,
  "entities": {
    "area": "baner",
    "city": "pune",
    "user_goal": "information",
    "zone": "west"
  },
  "is_vague": false,
  "primary_domain": "medical",
  "reason": "clear medical request",
  "secondary_domains": [],
  "should_continue_active_workflow": false,
  "should_preempt_active_workflow": false,
  "urgency_signal": "low"
}
Workflow Progress And Intents 
null
[]
Conversation Before And After 
{
  "active_state_machine": null,
  "active_workflow": null,
  "current_state": null,
  "entities": {},
  "messages": [],
  "phone": "919900000304",
  "service_actions": [],
  "started_at": "2026-03-16T22:30:37.908382+00:00",
  "triage_state": null,
  "updated_at": "2026-03-16T22:30:37.908386+00:00"
}
{
  "active_state_machine": "medical_triage",
  "active_workflow": null,
  "current_state": null,
  "entities": {},
  "messages": [
    {
      "role": "user",
      "source": null,
      "text": "Can you suggest a clinic nearby in Baner",
      "timestamp": "2026-03-16T22:30:38.237236+00:00"
    }
  ],
  "phone": "919900000304",
  "service_actions": [],
  "started_at": "2026-03-16T22:30:38.237231+00:00",
  "triage_state": null,
  "updated_at": "2026-03-16T22:30:38.237241+00:00"
}
Memory And Trace 
{
  "active_state_machine": null,
  "backend": "inmemory",
  "conversation_key_used": "919900000304",
  "conversation_keys": [
    "919900000304",
    "9900000304"
  ],
  "current_state": null,
  "loaded": true,
  "ping_ok": null,
  "redis_enabled": false,
  "thread_message_count": 0,
  "ttl_seconds_remaining": null
}
{
  "active_state_machine": "medical_triage",
  "backend": "inmemory",
  "conversation_key_used": "919900000304",
  "conversation_keys": [
    "919900000304",
    "9900000304"
  ],
  "current_state": null,
  "loaded": true,
  "ping_ok": null,
  "redis_enabled": false,
  "thread_message_count": 1,
  "ttl_seconds_remaining": null
}
[
  {
    "component": "brain",
    "payload": {
      "channel": "scenario_harness",
      "conversation_key": "919900000304",
      "message_text": "Can you suggest a clinic nearby in Baner",
      "normalized_phone": "919900000304",
      "phone": "919900000304"
    },
    "stage": "input",
    "summary": "Received normalized message envelope",
    "timestamp": "2026-03-16T22:30:37.922911+00:00"
  },
  {
    "component": "memory_store",
    "payload": {
      "active_state_machine": null,
      "backend": "inmemory",
      "current_state": null,
      "loaded": true,
      "message_count": 0,
      "storage_key": "919900000304"
    },
    "stage": "memory",
    "summary": "Loaded conversation memory",
    "timestamp": "2026-03-16T22:30:37.922915+00:00"
  },
  {
    "component": "brain",
    "payload": {
      "has_prior_triage_state": false,
      "prior_state_error": null
    },
    "stage": "state",
    "summary": "Parsed prior working state",
    "timestamp": "2026-03-16T22:30:37.922917+00:00"
  },
  {
    "component": "brain",
    "payload": {
      "compact_payload_keys": [
        "incoming_message"
      ],
      "has_conversation": true,
      "has_customer": false,
      "has_icp": false,
      "has_profile": false,
      "recent_import_count": 0,
      "storage_key": "919900000304"
    },
    "stage": "context",
    "summary": "Assembled SessionContext for the turn",
    "timestamp": "2026-03-16T22:30:37.922920+00:00"
  },
  {
    "component": "contact_resolver",
    "payload": {
      "confidence": 0.0,
      "customer_id": null,
      "display_name": null,
      "matched_phone": null,
      "status": "UNKNOWN"
    },
    "stage": "decision",
    "summary": "Resolved sender contact status",
    "timestamp": "2026-03-16T22:30:37.922923+00:00"
  },
  {
    "component": "turn_interpretation",
    "payload": {
      "confidence": 0.9,
      "contact_status": "UNKNOWN",
      "entities": {
        "area": "baner",
        "city": "pune",
        "user_goal": "information",
        "zone": "west"
      },
      "is_vague": false,
      "primary_domain": "medical",
      "reason": "clear medical request",
      "secondary_domains": [],
      "should_continue_active_workflow": false,
      "should_preempt_active_workflow": false,
      "urgency_signal": "low"
    },
    "stage": "decision",
    "summary": "Resolved turn interpretation deterministically",
    "timestamp": "2026-03-16T22:30:37.923069+00:00"
  },
  {
    "component": "profile_context",
    "payload": {
      "phone": "919900000304"
    },
    "stage": "context",
    "summary": "No customer matched provided phone",
    "timestamp": "2026-03-16T22:30:37.923073+00:00"
  },
  {
    "component": "entry_policy",
    "payload": {
      "allow_identity_question": true,
      "bypass_onboarding": false,
      "defer_existing_workflow": false,
      "preempt_active_workflow": false,
      "reason": "unknown contact with a clear non-urgent request can allow one light identity question later",
      "run_welcome_orientation": false,
      "selected_posture": "light_onboarding"
    },
    "stage": "decision",
    "summary": "Determined entry posture for this turn",
    "timestamp": "2026-03-16T22:30:37.923075+00:00"
  },
  {
    "component": "intent_parser",
    "payload": {
      "continue_active_workflow": false,
      "primary_intent": "clinic_request",
      "reason": "matched_keyword_rules",
      "resolved_state_machine": "medical_triage",
      "workflow_source": "intent_match"
    },
    "stage": "decision",
    "summary": "Resolved primary intent and state machine",
    "timestamp": "2026-03-16T22:30:37.923264+00:00"
  },
  {
    "component": "context_builder",
    "payload": {
      "active_workflow": "medical_triage",
      "background_fact_count": 0,
      "current_step": null,
      "foreground_fact_count": 3,
      "prompt_family": "medical",
      "purpose": "workflow_response"
    },
    "stage": "context",
    "summary": "Built workflow-response context slice for prompt assembly",
    "timestamp": "2026-03-16T22:30:37.923291+00:00"
  },
  {
    "component": "brain",
    "payload": {
      "current_state_before": null,
      "reason": "intent matched a registered state machine",
      "state_machine": "medical_triage"
    },
    "stage": "decision",
    "summary": "Executing state machine path",
    "timestamp": "2026-03-16T22:30:37.923323+00:00"
  },
  {
    "component": "medical_triage",
    "payload": {
      "compact_payload": {
        "incoming_message": "Can you suggest a clinic nearby in Baner",
        "prompt_family": "medical",
        "workflow_response_context": {
          "active_workflow": "medical_triage",
          "background_facts": [],
          "current_step": null,
          "entities": {
            "area": "baner",
            "city": "pune"
          },
          "foreground_facts": [
            {
              "confidence": 1.0,
              "freshness": "current_turn",
              "key": "active_workflow",
              "source": "routing",
              "use_mode": "direct",
              "value": "medical_triage"
            },
            {
              "confidence": 0.9,
              "freshness": "current_turn",
              "key": "area",
              "source": "turn_interpretation.entities",
              "use_mode": "direct",
              "value": "baner"
            },
            {
              "confidence": 0.9,
              "freshness": "current_turn",
              "key": "city",
              "source": "turn_interpretation.entities",
              "use_mode": "direct",
              "value": "pune"
            }
          ],
          "icp_guidance": {},
          "instruction_hints": [
            "Continue medical_triage at the current triage step. Prioritize the current symptom report, urgency, and recent thread details before any older profile history.",
            "Use direct facts from the active workflow and recent thread confidently.",
            "Treat confirm facts as historical references that must be verified before you assert them."
          ],
          "llm_required": true,
          "prompt_family": "medical",
          "purpose": "workflow_response",
          "recent_turns": [],
          "relationship_context": {},
          "service_history_context": {},
          "task_hint": "Continue medical_triage at the current triage step. Prioritize the current symptom report, urgency, and recent thread details before any older profile history."
        }
      },
      "current_state_before": null,
      "model": "gpt-5.4",
      "prior_triage_state": null,
      "prompt_family": "medical"
    },
    "stage": "llm_request",
    "summary": "Prepared medical triage prompt using current machine state, prior triage state, and compact customer context",
    "timestamp": "2026-03-16T22:30:38.237201+00:00"
  },
  {
    "component": "state_machine",
    "payload": {
      "clinic_routing_triggered": false,
      "current_state": null,
      "next_state": null,
      "service_action_count": 0
    },
    "stage": "response",
    "summary": "State machine produced response",
    "timestamp": "2026-03-16T22:30:38.237219+00:00"
  },
  {
    "component": "memory_store",
    "payload": {
      "active_state_machine": "medical_triage",
      "current_state": null,
      "message_count_after": 1,
      "storage_key": "919900000304"
    },
    "stage": "mutation",
    "summary": "Persisted updated conversation memory",
    "timestamp": "2026-03-16T22:30:38.238746+00:00"
  }
]
Turn 2
medical_triage medical_triage preempt_to_medical urgent_medical_preemption
Medical supersession Expectation failure
LIKELY BUG
User
Also can someone transport my cat there tomorrow morning
Assistant
—

What Changed This Turn

  • No material state change.

Expectation And Heuristic Results

Expectations
LIKELY BUG: workflow_name expected 'transport' but got 'medical_triage'
Routing And Interpretation 
{
  "confidence": 1.0,
  "reason_codes": [
    "preempt_to_medical",
    "urgent_medical_preemption"
  ],
  "route_type": "medical_triage",
  "workflow_name": "medical_triage"
}
{
  "confidence": 0.91,
  "entities": {
    "date_phrase": "tomorrow",
    "time_text": "morning"
  },
  "is_vague": false,
  "primary_domain": "transport",
  "reason": "new explicit domain supersedes active workflow",
  "secondary_domains": [],
  "should_continue_active_workflow": false,
  "should_preempt_active_workflow": true,
  "urgency_signal": "none"
}
Workflow Progress And Intents 
null
[]
Conversation Before And After 
{
  "active_state_machine": "medical_triage",
  "active_workflow": null,
  "current_state": null,
  "entities": {},
  "messages": [
    {
      "role": "user",
      "source": null,
      "text": "Can you suggest a clinic nearby in Baner",
      "timestamp": "2026-03-16T22:30:38.237236+00:00"
    }
  ],
  "phone": "919900000304",
  "service_actions": [],
  "started_at": "2026-03-16T22:30:38.237231+00:00",
  "triage_state": null,
  "updated_at": "2026-03-16T22:30:38.237241+00:00"
}
{
  "active_state_machine": "medical_triage",
  "active_workflow": null,
  "current_state": null,
  "entities": {},
  "messages": [
    {
      "role": "user",
      "source": null,
      "text": "Can you suggest a clinic nearby in Baner",
      "timestamp": "2026-03-16T22:30:38.237236+00:00"
    },
    {
      "role": "user",
      "source": null,
      "text": "Also can someone transport my cat there tomorrow morning",
      "timestamp": "2026-03-16T22:30:38.556519+00:00"
    }
  ],
  "phone": "919900000304",
  "service_actions": [],
  "started_at": "2026-03-16T22:30:38.237231+00:00",
  "triage_state": null,
  "updated_at": "2026-03-16T22:30:38.556523+00:00"
}
Memory And Trace 
{
  "active_state_machine": "medical_triage",
  "backend": "inmemory",
  "conversation_key_used": "919900000304",
  "conversation_keys": [
    "919900000304",
    "9900000304"
  ],
  "current_state": null,
  "loaded": true,
  "ping_ok": null,
  "redis_enabled": false,
  "thread_message_count": 1,
  "ttl_seconds_remaining": null
}
{
  "active_state_machine": "medical_triage",
  "backend": "inmemory",
  "conversation_key_used": "919900000304",
  "conversation_keys": [
    "919900000304",
    "9900000304"
  ],
  "current_state": null,
  "loaded": true,
  "ping_ok": null,
  "redis_enabled": false,
  "thread_message_count": 2,
  "ttl_seconds_remaining": null
}
[
  {
    "component": "brain",
    "payload": {
      "channel": "scenario_harness",
      "conversation_key": "919900000304",
      "message_text": "Also can someone transport my cat there tomorrow morning",
      "normalized_phone": "919900000304",
      "phone": "919900000304"
    },
    "stage": "input",
    "summary": "Received normalized message envelope",
    "timestamp": "2026-03-16T22:30:38.248555+00:00"
  },
  {
    "component": "memory_store",
    "payload": {
      "active_state_machine": "medical_triage",
      "backend": "inmemory",
      "current_state": null,
      "loaded": true,
      "message_count": 1,
      "storage_key": "919900000304"
    },
    "stage": "memory",
    "summary": "Loaded conversation memory",
    "timestamp": "2026-03-16T22:30:38.248562+00:00"
  },
  {
    "component": "brain",
    "payload": {
      "has_prior_triage_state": false,
      "prior_state_error": null
    },
    "stage": "state",
    "summary": "Parsed prior working state",
    "timestamp": "2026-03-16T22:30:38.248564+00:00"
  },
  {
    "component": "brain",
    "payload": {
      "compact_payload_keys": [
        "incoming_message"
      ],
      "has_conversation": true,
      "has_customer": false,
      "has_icp": false,
      "has_profile": false,
      "recent_import_count": 0,
      "storage_key": "919900000304"
    },
    "stage": "context",
    "summary": "Assembled SessionContext for the turn",
    "timestamp": "2026-03-16T22:30:38.248568+00:00"
  },
  {
    "component": "contact_resolver",
    "payload": {
      "confidence": 0.0,
      "customer_id": null,
      "display_name": null,
      "matched_phone": null,
      "status": "UNKNOWN"
    },
    "stage": "decision",
    "summary": "Resolved sender contact status",
    "timestamp": "2026-03-16T22:30:38.248573+00:00"
  },
  {
    "component": "turn_interpretation",
    "payload": {
      "confidence": 0.91,
      "contact_status": "UNKNOWN",
      "entities": {
        "date_phrase": "tomorrow",
        "time_text": "morning"
      },
      "is_vague": false,
      "primary_domain": "transport",
      "reason": "new explicit domain supersedes active workflow",
      "secondary_domains": [],
      "should_continue_active_workflow": false,
      "should_preempt_active_workflow": true,
      "urgency_signal": "none"
    },
    "stage": "decision",
    "summary": "Resolved turn interpretation deterministically",
    "timestamp": "2026-03-16T22:30:38.248788+00:00"
  },
  {
    "component": "profile_context",
    "payload": {
      "phone": "919900000304"
    },
    "stage": "context",
    "summary": "No customer matched provided phone",
    "timestamp": "2026-03-16T22:30:38.248794+00:00"
  },
  {
    "component": "entry_policy",
    "payload": {
      "allow_identity_question": false,
      "bypass_onboarding": true,
      "defer_existing_workflow": true,
      "preempt_active_workflow": true,
      "reason": "urgent medical risk overrides active workflow continuity",
      "run_welcome_orientation": false,
      "selected_posture": "preempt_to_medical"
    },
    "stage": "decision",
    "summary": "Determined entry posture for this turn",
    "timestamp": "2026-03-16T22:30:38.248796+00:00"
  },
  {
    "component": "entry_policy",
    "payload": {
      "forced_primary_intent": "medical_issue",
      "forced_selected_workflow": "medical_triage",
      "original_primary_intent": "medical_issue",
      "original_selected_workflow": "medical_triage",
      "reason": "urgent medical risk overrides active workflow continuity",
      "selected_posture": "preempt_to_medical"
    },
    "stage": "decision",
    "summary": "Entry posture overrode the initial route to force medical handling",
    "timestamp": "2026-03-16T22:30:38.248827+00:00"
  },
  {
    "component": "intent_parser",
    "payload": {
      "continue_active_workflow": false,
      "primary_intent": "medical_issue",
      "reason": "urgent medical risk overrides active workflow continuity",
      "resolved_state_machine": "medical_triage",
      "workflow_source": "intent_match"
    },
    "stage": "decision",
    "summary": "Resolved primary intent and state machine",
    "timestamp": "2026-03-16T22:30:38.248829+00:00"
  },
  {
    "component": "context_builder",
    "payload": {
      "active_workflow": "medical_triage",
      "background_fact_count": 0,
      "current_step": null,
      "foreground_fact_count": 1,
      "prompt_family": "medical",
      "purpose": "workflow_response"
    },
    "stage": "context",
    "summary": "Built workflow-response context slice for prompt assembly",
    "timestamp": "2026-03-16T22:30:38.248861+00:00"
  },
  {
    "component": "brain",
    "payload": {
      "current_state_before": null,
      "reason": "intent matched a registered state machine",
      "state_machine": "medical_triage"
    },
    "stage": "decision",
    "summary": "Executing state machine path",
    "timestamp": "2026-03-16T22:30:38.248901+00:00"
  },
  {
    "component": "medical_triage",
    "payload": {
      "compact_payload": {
        "incoming_message": "Also can someone transport my cat there tomorrow morning",
        "prompt_family": "medical",
        "workflow_response_context": {
          "active_workflow": "medical_triage",
          "background_facts": [],
          "current_step": null,
          "entities": {},
          "foreground_facts": [
            {
              "confidence": 1.0,
              "freshness": "current_turn",
              "key": "active_workflow",
              "source": "routing",
              "use_mode": "direct",
              "value": "medical_triage"
            }
          ],
          "icp_guidance": {},
          "instruction_hints": [
            "Continue medical_triage at the current triage step. Prioritize the current symptom report, urgency, and recent thread details before any older profile history.",
            "Use direct facts from the active workflow and recent thread confidently.",
            "Treat confirm facts as historical references that must be verified before you assert them."
          ],
          "llm_required": true,
          "prompt_family": "medical",
          "purpose": "workflow_response",
          "recent_turns": [
            {
              "role": "user",
              "text": "Can you suggest a clinic nearby in Baner"
            }
          ],
          "relationship_context": {},
          "service_history_context": {},
          "task_hint": "Continue medical_triage at the current triage step. Prioritize the current symptom report, urgency, and recent thread details before any older profile history."
        }
      },
      "current_state_before": null,
      "model": "gpt-5.4",
      "prior_triage_state": null,
      "prompt_family": "medical"
    },
    "stage": "llm_request",
    "summary": "Prepared medical triage prompt using current machine state, prior triage state, and compact customer context",
    "timestamp": "2026-03-16T22:30:38.556483+00:00"
  },
  {
    "component": "state_machine",
    "payload": {
      "clinic_routing_triggered": false,
      "current_state": null,
      "next_state": null,
      "service_action_count": 0
    },
    "stage": "response",
    "summary": "State machine produced response",
    "timestamp": "2026-03-16T22:30:38.556501+00:00"
  },
  {
    "component": "memory_store",
    "payload": {
      "active_state_machine": "medical_triage",
      "current_state": null,
      "message_count_after": 2,
      "storage_key": "919900000304"
    },
    "stage": "mutation",
    "summary": "Persisted updated conversation memory",
    "timestamp": "2026-03-16T22:30:38.556738+00:00"
  }
]