Raahat Scenario Harness

Curated conversation replay against the real runtime with stepwise diagnostics, readable deltas, and deterministic checks.

One Word Slot Answers

continuity_one_word_slot_answers · continuity · 2/2 turns executed

Scenario Summary

IDcontinuity_one_word_slot_answers
TitleOne Word Slot Answers
Categorycontinuity
Turns2/2
Run Statecompleted
terse boarding multi_turn

Seed State

UserUnknown user
Phone919900000401
Seed Workflowboarding
Seed State Machine

Current Runtime State

Routeboarding
Workflowboarding
State Machine
Stagequalifying

Checks And Lifecycle

Expectations Passed2
Expectations Failed0
Heuristic Warnings0
Sticky continuation
Final assessmentOperationally incomplete: unresolved fields remain (location).

Final Outcome

Completed

Completed vs partial run state.

Final Route

boarding

Current route at the end of the replay.

Final Workflow / State

boarding

Workflow preferred; state machine shown if no workflow is active.

Final Stage

qualifying

Resolved from workflow progress or active snapshot.

Expectation Totals

Pass 2 Fail 0

Heuristic Totals

Warnings 0

Missing Fields

location

Lifecycle Events Seen

Sticky continuation
Turn 1
boarding boarding qualifying active_workflow_snapshot follow_up active_workflow_snapshot
Sticky continuation New slots filled Memory updated Stayed on boarding
PASS
User
Friday
Assistant
Got it. How many cats is this for?

What Changed This Turn

  • Active workflow remained boarding.
  • New slot filled: check_in_date=2026-03-20.
  • New slot filled: last_question_key=pet_count.
  • New slot filled: raw_date_text=friday.
  • New slot filled: stage=qualifying.
  • Memory updated with check_in_date.
  • Memory updated with date_phrase.
  • Memory updated with last_non_medical_domain.
  • Memory updated with request_kind.
  • Snapshot version incremented to 2.

Expectation And Heuristic Results

Expectations
PASS: workflow_name matched expected value 'boarding'
Routing And Interpretation
{
  "confidence": 0.89,
  "reason_codes": [
    "active_workflow_snapshot",
    "follow_up",
    "active_workflow_snapshot",
    "follow_up_slot_fill"
  ],
  "route_type": "boarding",
  "workflow_name": "boarding"
}
{
  "confidence": 0.89,
  "entities": {
    "date_phrase": "friday"
  },
  "is_vague": false,
  "primary_domain": "boarding",
  "reason": "slot-like follow-up matches active workflow",
  "secondary_domains": [],
  "should_continue_active_workflow": true,
  "should_preempt_active_workflow": false,
  "urgency_signal": "none"
}
Workflow Progress And Intents
{
  "missing_fields": [
    "pet_count",
    "location"
  ],
  "next_question": "Got it. How many cats is this for?",
  "notes": [],
  "stage": "qualifying",
  "state_payload": {
    "check_in_date": "2026-03-20",
    "check_in_time_window": null,
    "check_out_date": null,
    "check_out_time_window": null,
    "duration_nights": null,
    "last_question_key": "pet_count",
    "medical_flags": [],
    "pet_count": null,
    "pet_names": [],
    "pickup_area": null,
    "pickup_required": null,
    "raw_date_text": "friday",
    "raw_duration_text": null,
    "request_kind": "availability",
    "service_area": null,
    "special_needs": [],
    "stage": "qualifying"
  },
  "status": "collecting",
  "summary": "2026-03-20",
  "superseded_by": null,
  "workflow_name": "boarding"
}
[]
Conversation Before And After
{
  "active_state_machine": null,
  "active_workflow": {
    "last_question_key": "dates",
    "missing_fields": [
      "dates",
      "pet_count"
    ],
    "next_question": null,
    "stage": "qualifying",
    "started_at": "2026-03-16T22:32:48.193010+00:00",
    "state_payload": {
      "request_kind": "availability"
    },
    "status": "collecting",
    "summary": null,
    "superseded_by": null,
    "updated_at": "2026-03-16T22:32:48.193014+00:00",
    "version": 1,
    "workflow_name": "boarding"
  },
  "current_state": null,
  "entities": {},
  "messages": [],
  "phone": "919900000401",
  "service_actions": [],
  "started_at": "2026-03-16T22:32:48.193037+00:00",
  "triage_state": null,
  "updated_at": "2026-03-16T22:32:48.193038+00:00"
}
{
  "active_state_machine": null,
  "active_workflow": {
    "last_question_key": "pet_count",
    "missing_fields": [
      "pet_count",
      "location"
    ],
    "next_question": "Got it. How many cats is this for?",
    "stage": "qualifying",
    "started_at": "2026-03-16T22:32:48.193010+00:00",
    "state_payload": {
      "check_in_date": "2026-03-20",
      "check_in_time_window": null,
      "check_out_date": null,
      "check_out_time_window": null,
      "duration_nights": null,
      "last_question_key": "pet_count",
      "medical_flags": [],
      "pet_count": null,
      "pet_names": [],
      "pickup_area": null,
      "pickup_required": null,
      "raw_date_text": "friday",
      "raw_duration_text": null,
      "request_kind": "availability",
      "service_area": null,
      "special_needs": [],
      "stage": "qualifying"
    },
    "status": "collecting",
    "summary": "2026-03-20",
    "superseded_by": null,
    "updated_at": "2026-03-16T22:32:48.213226+00:00",
    "version": 2,
    "workflow_name": "boarding"
  },
  "current_state": "QUALIFYING",
  "entities": {
    "check_in_date": "2026-03-20",
    "date_phrase": "friday",
    "last_non_medical_domain": "boarding",
    "request_kind": "availability"
  },
  "messages": [
    {
      "role": "user",
      "source": null,
      "text": "Friday",
      "timestamp": "2026-03-16T22:32:48.213271+00:00"
    },
    {
      "role": "assistant",
      "source": "B",
      "text": "Got it. How many cats is this for?",
      "timestamp": "2026-03-16T22:32:48.213273+00:00"
    }
  ],
  "phone": "919900000401",
  "service_actions": [],
  "started_at": "2026-03-16T22:32:48.213266+00:00",
  "triage_state": null,
  "updated_at": "2026-03-16T22:32:48.213293+00:00"
}
Memory And Trace
{
  "active_state_machine": null,
  "backend": "inmemory",
  "conversation_key_used": "919900000401",
  "conversation_keys": [
    "919900000401",
    "9900000401"
  ],
  "current_state": null,
  "loaded": true,
  "ping_ok": null,
  "redis_enabled": false,
  "thread_message_count": 0,
  "ttl_seconds_remaining": null
}
{
  "active_state_machine": null,
  "backend": "inmemory",
  "conversation_key_used": "919900000401",
  "conversation_keys": [
    "919900000401",
    "9900000401"
  ],
  "current_state": "QUALIFYING",
  "loaded": true,
  "ping_ok": null,
  "redis_enabled": false,
  "thread_message_count": 2,
  "ttl_seconds_remaining": null
}
[
  {
    "component": "brain",
    "payload": {
      "channel": "scenario_harness",
      "conversation_key": "919900000401",
      "message_text": "Friday",
      "normalized_phone": "919900000401",
      "phone": "919900000401"
    },
    "stage": "input",
    "summary": "Received normalized message envelope",
    "timestamp": "2026-03-16T22:32:48.212705+00:00"
  },
  {
    "component": "memory_store",
    "payload": {
      "active_state_machine": null,
      "backend": "inmemory",
      "current_state": null,
      "loaded": true,
      "message_count": 0,
      "storage_key": "919900000401"
    },
    "stage": "memory",
    "summary": "Loaded conversation memory",
    "timestamp": "2026-03-16T22:32:48.212713+00:00"
  },
  {
    "component": "brain",
    "payload": {
      "has_prior_triage_state": false,
      "prior_state_error": null
    },
    "stage": "state",
    "summary": "Parsed prior working state",
    "timestamp": "2026-03-16T22:32:48.212715+00:00"
  },
  {
    "component": "brain",
    "payload": {
      "compact_payload_keys": [
        "incoming_message"
      ],
      "has_conversation": true,
      "has_customer": false,
      "has_icp": false,
      "has_profile": false,
      "recent_import_count": 0,
      "storage_key": "919900000401"
    },
    "stage": "context",
    "summary": "Assembled SessionContext for the turn",
    "timestamp": "2026-03-16T22:32:48.212719+00:00"
  },
  {
    "component": "contact_resolver",
    "payload": {
      "confidence": 0.0,
      "customer_id": null,
      "display_name": null,
      "matched_phone": null,
      "status": "UNKNOWN"
    },
    "stage": "decision",
    "summary": "Resolved sender contact status",
    "timestamp": "2026-03-16T22:32:48.212724+00:00"
  },
  {
    "component": "turn_interpretation",
    "payload": {
      "confidence": 0.89,
      "contact_status": "UNKNOWN",
      "entities": {
        "date_phrase": "friday"
      },
      "is_vague": false,
      "primary_domain": "boarding",
      "reason": "slot-like follow-up matches active workflow",
      "secondary_domains": [],
      "should_continue_active_workflow": true,
      "should_preempt_active_workflow": false,
      "urgency_signal": "none"
    },
    "stage": "decision",
    "summary": "Resolved turn interpretation deterministically",
    "timestamp": "2026-03-16T22:32:48.212875+00:00"
  },
  {
    "component": "profile_context",
    "payload": {
      "phone": "919900000401"
    },
    "stage": "context",
    "summary": "No customer matched provided phone",
    "timestamp": "2026-03-16T22:32:48.212882+00:00"
  },
  {
    "component": "entry_policy",
    "payload": {
      "allow_identity_question": false,
      "bypass_onboarding": true,
      "defer_existing_workflow": false,
      "preempt_active_workflow": false,
      "reason": "existing active workflow should continue unless urgency preempts it",
      "run_welcome_orientation": false,
      "selected_posture": "continue_active"
    },
    "stage": "decision",
    "summary": "Determined entry posture for this turn",
    "timestamp": "2026-03-16T22:32:48.212884+00:00"
  },
  {
    "component": "intent_parser",
    "payload": {
      "continue_active_workflow": true,
      "primary_intent": "boarding_request",
      "reason": "continue_active_workflow_snapshot_from_follow_up",
      "resolved_state_machine": "boarding",
      "workflow_source": "active_workflow"
    },
    "stage": "decision",
    "summary": "Resolved primary intent and state machine",
    "timestamp": "2026-03-16T22:32:48.212933+00:00"
  },
  {
    "component": "context_builder",
    "payload": {
      "active_workflow": "boarding",
      "background_fact_count": 0,
      "current_step": "qualifying",
      "foreground_fact_count": 3,
      "prompt_family": "general",
      "purpose": "workflow_response"
    },
    "stage": "context",
    "summary": "Built workflow-response context slice for prompt assembly",
    "timestamp": "2026-03-16T22:32:48.212971+00:00"
  },
  {
    "component": "workflow",
    "payload": {
      "action_intents": [],
      "missing_fields": [
        "pet_count",
        "location"
      ],
      "next_question": "Got it. How many cats is this for?",
      "stage": "qualifying",
      "status": "collecting",
      "summary": "2026-03-20",
      "workflow_name": "boarding"
    },
    "stage": "decision",
    "summary": "Executed deterministic workflow handler",
    "timestamp": "2026-03-16T22:32:48.213244+00:00"
  },
  {
    "component": "memory_store",
    "payload": {
      "active_state_machine": null,
      "current_state": "QUALIFYING",
      "message_count_after": 2,
      "storage_key": "919900000401"
    },
    "stage": "mutation",
    "summary": "Persisted updated conversation memory",
    "timestamp": "2026-03-16T22:32:48.213703+00:00"
  }
]
Turn 2
boarding boarding qualifying active_workflow_snapshot follow_up active_workflow_snapshot
Sticky continuation New slots filled Memory updated Stayed on boarding
PASS PASS PASS
User
Two
Assistant
Got it. What area would pickup or drop be from?

What Changed This Turn

  • Active workflow remained boarding.
  • New slot filled: pet_count=2.
  • Memory updated with cat_count.
  • Snapshot version incremented to 3.

Expectation And Heuristic Results

Expectations
PASS: workflow_name matched expected value 'boarding'
Heuristics
PASS: Workflow avoided repeating the same question PASS: Active workflow remained sticky across terse follow-up
Routing And Interpretation
{
  "confidence": 0.92,
  "reason_codes": [
    "active_workflow_snapshot",
    "follow_up",
    "active_workflow_snapshot",
    "follow_up_slot_fill"
  ],
  "route_type": "boarding",
  "workflow_name": "boarding"
}
{
  "confidence": 0.92,
  "entities": {
    "cat_count": 2
  },
  "is_vague": false,
  "primary_domain": "boarding",
  "reason": "fills requested cat_count for active workflow",
  "secondary_domains": [],
  "should_continue_active_workflow": true,
  "should_preempt_active_workflow": false,
  "urgency_signal": "none"
}
Workflow Progress And Intents
{
  "missing_fields": [
    "location"
  ],
  "next_question": "Got it. What area would pickup or drop be from?",
  "notes": [],
  "stage": "qualifying",
  "state_payload": {
    "check_in_date": "2026-03-20",
    "check_in_time_window": null,
    "check_out_date": null,
    "check_out_time_window": null,
    "duration_nights": null,
    "last_question_key": "location",
    "medical_flags": [],
    "pet_count": 2,
    "pet_names": [],
    "pickup_area": null,
    "pickup_required": null,
    "raw_date_text": "friday",
    "raw_duration_text": null,
    "request_kind": "availability",
    "service_area": null,
    "special_needs": [],
    "stage": "qualifying"
  },
  "status": "collecting",
  "summary": "2 cats, 2026-03-20",
  "superseded_by": null,
  "workflow_name": "boarding"
}
[]
Conversation Before And After
{
  "active_state_machine": null,
  "active_workflow": {
    "last_question_key": "pet_count",
    "missing_fields": [
      "pet_count",
      "location"
    ],
    "next_question": "Got it. How many cats is this for?",
    "stage": "qualifying",
    "started_at": "2026-03-16T22:32:48.193010+00:00",
    "state_payload": {
      "check_in_date": "2026-03-20",
      "check_in_time_window": null,
      "check_out_date": null,
      "check_out_time_window": null,
      "duration_nights": null,
      "last_question_key": "pet_count",
      "medical_flags": [],
      "pet_count": null,
      "pet_names": [],
      "pickup_area": null,
      "pickup_required": null,
      "raw_date_text": "friday",
      "raw_duration_text": null,
      "request_kind": "availability",
      "service_area": null,
      "special_needs": [],
      "stage": "qualifying"
    },
    "status": "collecting",
    "summary": "2026-03-20",
    "superseded_by": null,
    "updated_at": "2026-03-16T22:32:48.213226+00:00",
    "version": 2,
    "workflow_name": "boarding"
  },
  "current_state": "QUALIFYING",
  "entities": {
    "check_in_date": "2026-03-20",
    "date_phrase": "friday",
    "last_non_medical_domain": "boarding",
    "request_kind": "availability"
  },
  "messages": [
    {
      "role": "user",
      "source": null,
      "text": "Friday",
      "timestamp": "2026-03-16T22:32:48.213271+00:00"
    },
    {
      "role": "assistant",
      "source": "B",
      "text": "Got it. How many cats is this for?",
      "timestamp": "2026-03-16T22:32:48.213273+00:00"
    }
  ],
  "phone": "919900000401",
  "service_actions": [],
  "started_at": "2026-03-16T22:32:48.213266+00:00",
  "triage_state": null,
  "updated_at": "2026-03-16T22:32:48.213293+00:00"
}
{
  "active_state_machine": null,
  "active_workflow": {
    "last_question_key": "location",
    "missing_fields": [
      "location"
    ],
    "next_question": "Got it. What area would pickup or drop be from?",
    "stage": "qualifying",
    "started_at": "2026-03-16T22:32:48.193010+00:00",
    "state_payload": {
      "check_in_date": "2026-03-20",
      "check_in_time_window": null,
      "check_out_date": null,
      "check_out_time_window": null,
      "duration_nights": null,
      "last_question_key": "location",
      "medical_flags": [],
      "pet_count": 2,
      "pet_names": [],
      "pickup_area": null,
      "pickup_required": null,
      "raw_date_text": "friday",
      "raw_duration_text": null,
      "request_kind": "availability",
      "service_area": null,
      "special_needs": [],
      "stage": "qualifying"
    },
    "status": "collecting",
    "summary": "2 cats, 2026-03-20",
    "superseded_by": null,
    "updated_at": "2026-03-16T22:32:48.217042+00:00",
    "version": 3,
    "workflow_name": "boarding"
  },
  "current_state": "QUALIFYING",
  "entities": {
    "cat_count": 2,
    "check_in_date": "2026-03-20",
    "date_phrase": "friday",
    "last_non_medical_domain": "boarding",
    "request_kind": "availability"
  },
  "messages": [
    {
      "role": "user",
      "source": null,
      "text": "Friday",
      "timestamp": "2026-03-16T22:32:48.213271+00:00"
    },
    {
      "role": "assistant",
      "source": "B",
      "text": "Got it. How many cats is this for?",
      "timestamp": "2026-03-16T22:32:48.213273+00:00"
    },
    {
      "role": "user",
      "source": null,
      "text": "Two",
      "timestamp": "2026-03-16T22:32:48.217223+00:00"
    },
    {
      "role": "assistant",
      "source": "B",
      "text": "Got it. What area would pickup or drop be from?",
      "timestamp": "2026-03-16T22:32:48.217227+00:00"
    }
  ],
  "phone": "919900000401",
  "service_actions": [],
  "started_at": "2026-03-16T22:32:48.213266+00:00",
  "triage_state": null,
  "updated_at": "2026-03-16T22:32:48.217292+00:00"
}
Memory And Trace
{
  "active_state_machine": null,
  "backend": "inmemory",
  "conversation_key_used": "919900000401",
  "conversation_keys": [
    "919900000401",
    "9900000401"
  ],
  "current_state": "QUALIFYING",
  "loaded": true,
  "ping_ok": null,
  "redis_enabled": false,
  "thread_message_count": 2,
  "ttl_seconds_remaining": null
}
{
  "active_state_machine": null,
  "backend": "inmemory",
  "conversation_key_used": "919900000401",
  "conversation_keys": [
    "919900000401",
    "9900000401"
  ],
  "current_state": "QUALIFYING",
  "loaded": true,
  "ping_ok": null,
  "redis_enabled": false,
  "thread_message_count": 4,
  "ttl_seconds_remaining": null
}
[
  {
    "component": "brain",
    "payload": {
      "channel": "scenario_harness",
      "conversation_key": "919900000401",
      "message_text": "Two",
      "normalized_phone": "919900000401",
      "phone": "919900000401"
    },
    "stage": "input",
    "summary": "Received normalized message envelope",
    "timestamp": "2026-03-16T22:32:48.216538+00:00"
  },
  {
    "component": "memory_store",
    "payload": {
      "active_state_machine": null,
      "backend": "inmemory",
      "current_state": "QUALIFYING",
      "loaded": true,
      "message_count": 2,
      "storage_key": "919900000401"
    },
    "stage": "memory",
    "summary": "Loaded conversation memory",
    "timestamp": "2026-03-16T22:32:48.216542+00:00"
  },
  {
    "component": "brain",
    "payload": {
      "has_prior_triage_state": false,
      "prior_state_error": null
    },
    "stage": "state",
    "summary": "Parsed prior working state",
    "timestamp": "2026-03-16T22:32:48.216544+00:00"
  },
  {
    "component": "brain",
    "payload": {
      "compact_payload_keys": [
        "incoming_message"
      ],
      "has_conversation": true,
      "has_customer": false,
      "has_icp": false,
      "has_profile": false,
      "recent_import_count": 0,
      "storage_key": "919900000401"
    },
    "stage": "context",
    "summary": "Assembled SessionContext for the turn",
    "timestamp": "2026-03-16T22:32:48.216547+00:00"
  },
  {
    "component": "contact_resolver",
    "payload": {
      "confidence": 0.0,
      "customer_id": null,
      "display_name": null,
      "matched_phone": null,
      "status": "UNKNOWN"
    },
    "stage": "decision",
    "summary": "Resolved sender contact status",
    "timestamp": "2026-03-16T22:32:48.216551+00:00"
  },
  {
    "component": "turn_interpretation",
    "payload": {
      "confidence": 0.92,
      "contact_status": "UNKNOWN",
      "entities": {
        "cat_count": 2
      },
      "is_vague": false,
      "primary_domain": "boarding",
      "reason": "fills requested cat_count for active workflow",
      "secondary_domains": [],
      "should_continue_active_workflow": true,
      "should_preempt_active_workflow": false,
      "urgency_signal": "none"
    },
    "stage": "decision",
    "summary": "Resolved turn interpretation deterministically",
    "timestamp": "2026-03-16T22:32:48.216651+00:00"
  },
  {
    "component": "profile_context",
    "payload": {
      "phone": "919900000401"
    },
    "stage": "context",
    "summary": "No customer matched provided phone",
    "timestamp": "2026-03-16T22:32:48.216655+00:00"
  },
  {
    "component": "entry_policy",
    "payload": {
      "allow_identity_question": false,
      "bypass_onboarding": true,
      "defer_existing_workflow": false,
      "preempt_active_workflow": false,
      "reason": "existing active workflow should continue unless urgency preempts it",
      "run_welcome_orientation": false,
      "selected_posture": "continue_active"
    },
    "stage": "decision",
    "summary": "Determined entry posture for this turn",
    "timestamp": "2026-03-16T22:32:48.216656+00:00"
  },
  {
    "component": "intent_parser",
    "payload": {
      "continue_active_workflow": true,
      "primary_intent": "boarding_request",
      "reason": "continue_active_workflow_snapshot_from_follow_up",
      "resolved_state_machine": "boarding",
      "workflow_source": "active_workflow"
    },
    "stage": "decision",
    "summary": "Resolved primary intent and state machine",
    "timestamp": "2026-03-16T22:32:48.216693+00:00"
  },
  {
    "component": "context_builder",
    "payload": {
      "active_workflow": "boarding",
      "background_fact_count": 0,
      "current_step": "QUALIFYING",
      "foreground_fact_count": 4,
      "prompt_family": "general",
      "purpose": "workflow_response"
    },
    "stage": "context",
    "summary": "Built workflow-response context slice for prompt assembly",
    "timestamp": "2026-03-16T22:32:48.216720+00:00"
  },
  {
    "component": "workflow",
    "payload": {
      "action_intents": [],
      "missing_fields": [
        "location"
      ],
      "next_question": "Got it. What area would pickup or drop be from?",
      "stage": "qualifying",
      "status": "collecting",
      "summary": "2 cats, 2026-03-20",
      "workflow_name": "boarding"
    },
    "stage": "decision",
    "summary": "Executed deterministic workflow handler",
    "timestamp": "2026-03-16T22:32:48.217058+00:00"
  },
  {
    "component": "memory_store",
    "payload": {
      "active_state_machine": null,
      "current_state": "QUALIFYING",
      "message_count_after": 4,
      "storage_key": "919900000401"
    },
    "stage": "mutation",
    "summary": "Persisted updated conversation memory",
    "timestamp": "2026-03-16T22:32:48.218430+00:00"
  }
]