{ "title": "Questionnaire depth cost benchmark: default vs reduced questionnaire on a 100-respondent study", "measured_at": "2026-09-24", "platform": "MoeVox, local development deployment (Gemini for questionnaire generation and open-text answers, Jev for structured answers, MongoDB respondent pool)", "engines": { "questionnaire_generation": "gemini-3.5-flash-lite", "structured_sampling_answers": "Jev (TYPESAFE API), used when the report language is English", "open_text_sampling_answers": "gemini-3.5-flash-lite (open-text questions are not routed to Jev)", "respondent_profile_enrichment": "gemini-3.5-flash-lite", "sampling_concurrency": 30 }, "research_pool": { "respondent_profiles_total": 114423, "respondent_profiles_missing_ai_attributes": 111914, "note": "Respondent profiles missing traits / persona latents / motivations trigger one enrichment call per sampled respondent in the default configuration." }, "design": { "held_constant": [ "research question: which of three pricing plans to launch first", "three candidate options", "report language: English", "sampling concurrency: 30", "respondent pool and audience filters", "price: 1 credit per completed respondent" ], "changed": [ "reduced questionnaire mode: no open-text question, no support questions, respondent-profile enrichment skipped" ], "iterations": [ { "iteration": 1, "respondents_per_run": 6, "report_snapshot_stage_measured": false }, { "iteration": 2, "respondents_per_run": 6, "report_snapshot_stage_measured": true }, { "iteration": 3, "respondents_per_run": 100, "report_snapshot_stage_measured": true } ], "credits_consumed_total": 224, "note_on_shapes": "Iterations 1 and 2 record per-respondent averages only (avg_ms, enrichment_avg_ms, enrichment_count); iteration 3 records the full distribution (avg_ms, p95_ms, max_ms, the enrichment split and the winner/follow-up split)." }, "instrument": { "default": { "questions": 11, "core_questions": 3, "open_text_questions": 1, "support_questions": 7, "roles": [ "winner", "driver", "risk_structured", "risk_open", "pricing_reaction", "absolute_validation", "adoption_intent", "recommendation_support", "trust", "clarity", "fit" ] }, "fast": { "questions": 3, "core_questions": 3, "open_text_questions": 0, "support_questions": 0, "roles": [ "winner", "driver", "risk_structured" ] } }, "model_calls_per_respondent": { "default": { "total": 4, "winner_call": 1, "structured_call": 1, "open_text_call": 1, "enrichment_call": "1 for respondents whose profile still lacks AI attributes (97 of 100 in iteration 3)", "max_questions_in_follow_up_calls": 10 }, "fast": { "total": 2, "winner_call": 1, "structured_call": 1, "open_text_call": 0, "enrichment_call": 0, "max_questions_in_follow_up_calls": 2 } }, "results": { "iteration_1_respondents_6": { "default": { "questions": 11, "generation_ms": 15121, "generation_model_calls": 2, "sampling_ms": 22328, "sampling_model_calls": { "gemini": 13, "jev": 12 }, "report_ms": null, "successful_respondents": 6, "failed_respondents": 0, "per_respondent": { "avg_ms": 9077, "enrichment_avg_ms": 4789, "enrichment_count": 6 } }, "fast": { "questions": 3, "generation_ms": 3029, "generation_model_calls": 3, "sampling_ms": 4129, "sampling_model_calls": { "gemini": 2, "jev": 12 }, "report_ms": null, "successful_respondents": 6, "failed_respondents": 0, "per_respondent": { "avg_ms": 2048, "enrichment_avg_ms": null, "enrichment_count": 0 } } }, "iteration_2_respondents_6": { "default": { "questions": 11, "generation_ms": 6060, "generation_model_calls": 1, "sampling_ms": 10309, "sampling_model_calls": { "gemini": 12, "jev": 12 }, "report_ms": 12953, "report_model_calls": 6, "total_ms": 29322, "successful_respondents": 6, "failed_respondents": 0, "per_respondent": { "avg_ms": 6875, "enrichment_avg_ms": 3396, "enrichment_count": 5 } }, "fast": { "questions": 3, "generation_ms": 4117, "generation_model_calls": 1, "sampling_ms": 33748, "sampling_model_calls": { "gemini": 1, "jev": 12 }, "report_ms": 7107, "report_model_calls": 3, "total_ms": 44972, "successful_respondents": 6, "failed_respondents": 0, "per_respondent": { "avg_ms": 11732, "enrichment_avg_ms": null, "enrichment_count": 0 } } }, "iteration_3_respondents_100": { "default": { "questions": 11, "generation_ms": 10254, "generation_model_calls": 2, "generation_draft_prompt_chars": 12387, "generation_draft_response_chars": 5460, "sampling_ms": 44086, "sampling_wall_ms_from_database": 42999, "sampling_model_calls": { "gemini": 213, "jev": 200 }, "report_ms": 19383, "report_model_calls": 9, "total_ms": 73723, "successful_respondents": 100, "failed_respondents": 0, "per_respondent": { "avg_ms": 8486, "p95_ms": 18692, "max_ms": 42752, "enrichment_avg_ms": 4096, "enrichment_p95_ms": 11912, "enrichment_max_ms": 34131, "enrichment_count": 97, "winner_call_avg_ms": 1271, "follow_up_call_avg_ms": 3147, "follow_up_call_requests": 200, "max_questions_in_follow_up_calls": 10 } }, "fast": { "questions": 3, "generation_ms": 4054, "generation_model_calls": 1, "generation_draft_prompt_chars": 11276, "generation_draft_response_chars": 1946, "sampling_ms": 16366, "sampling_wall_ms_from_database": 15174, "sampling_model_calls": { "gemini": 1, "jev": 200 }, "report_ms": 5342, "report_model_calls": 3, "total_ms": 25762, "successful_respondents": 100, "failed_respondents": 0, "per_respondent": { "avg_ms": 1859, "p95_ms": 4354, "max_ms": 14698, "enrichment_avg_ms": null, "enrichment_p95_ms": null, "enrichment_max_ms": null, "enrichment_count": 0, "winner_call_avg_ms": 999, "follow_up_call_avg_ms": 792, "follow_up_call_requests": 100, "max_questions_in_follow_up_calls": 2 } } } }, "model_call_totals_iteration_3": { "default": { "gemini": 224, "jev": 200 }, "fast": { "gemini": 5, "jev": 200 } }, "notes": [ "Wall-clock sampling time is bounded by the slowest respondent in a batch when concurrency (30) is larger than the batch, so the 6-respondent iterations reverse direction (iteration 1: fast 4.1s vs default 22.3s; iteration 2: fast 33.7s vs default 10.3s). Per-respondent chain averages are the stable statistic.", "The default run's 213 Gemini calls decompose into 100 open-text calls, 97 enrichment calls and 16 repair/retry calls.", "The fast run's single Gemini call is one repaired answer; the other 99 respondents were answered entirely by Jev.", "Report snapshot calls are not reduced to zero in the fast configuration: segment and summary work still runs. The open-text theme merging that the default run performs disappears with the open-text question.", "Credits: 1 credit per completed respondent. Iteration 3 cost 100 credits per run; all three iterations cost 224 credits in total.", "These numbers come from one research question, one language and one deployment. They describe the shape of the saving, not a guaranteed ratio for other studies." ] }