{ "generatedAt": "2026-10-26T17:03:03.498676+00:00", "window": { "start": "2026-10-12T16:42:47.757Z", "end": "2026-10-26T16:42:47.757Z" }, "method": { "selection": "Top 20 per gender by distinct real registered callers in positive-duration private call transcripts over last 14 days; public non-retired characters with active Fish voice. Tie-break by call count then character id.", "generation": "40 distinct existing Fish voice references × three fixed English scripts × two models. Both models receive the same voice, text, chunking, speed 1.0, temperature 0.7, top_p 0.7, balanced latency and quality guard. Unmodified mono 24 kHz PCM is wrapped in WAV; no gain, trimming or resampling.", "judge": "Gemini 3.1 Pro receives an opaque pair code, the script/delivery and A/B audio only. Each recording pair is judged again with presentation order reversed. Naturalness is the predeclared primary score: average six ratings per model per character, then give each of 40 characters equal weight. The descriptive 95% interval uses 5,000 paired character bootstrap samples stratified by gender.", "limitations": "One AI judge and one synthesis per model/voice/script. The three English scripts do not establish multilingual quality, human preference or production retention. No reference audio was supplied, so this is not a voice-identity fidelity test. Order-reversed reviews are reliability checks of the same recordings.", "notes": [ "Judge receives opaque pair code, task directions, exact text and two audio files; no model/character names, portrait, gender, popularity, timings or file paths. Same audio reviewed in reverse order in separate requests. Orchestrator knows mapping; not independent human double-blind.", "Confidence values are the mean of two uncalibrated model self-reports, not probabilities of a real-world win.", "Selection, scripts, masking, controls and aggregation rules were written before synthesis. Exact transport settings and retry details were appended during generation after the pilot; scoring criteria and samples were not changed.", "All 40 selected characters remain in the gallery; 0 sample-reviews were unassessable. A metric requires all six scores per model per character; missing ratings are never skipped or imputed.", "Masked A/B results and hashes were frozen before the aggregation code decoded model labels.", "Identical-audio calibration: 16 ties, 0 non-tie preferences and 0 unassessable reviews out of 16.", "Direct cold WebSocket requests from this test host; no warm connection pool, not end-to-end production call latency.", "Synthesis retries: 0 failed attempts across 0 retried samples; all 240 final WAVs decoded successfully.", "Provider model labels identify the requested Fish header. Response model identity is included only if the provider returned it; no claim of independent backend attestation.", "No production voice settings or existing rollout experiments were changed." ] }, "summary": { "characters": 40, "voiceReferences": 40, "genderBalance": { "Male": 20, "Female": 20 }, "pairs": 120, "judgments": 240, "calibrationJudgments": 16, "totalJudgments": 256, "samples": 240, "modelWins": { "s2-pro": 27, "s2.1-pro": 20 }, "ties": 9, "inconsistent": 64, "unassessableReviews": 0, "unassessableCases": [], "protocolNotes": "Selection, scripts, masking, controls and aggregation rules were written before synthesis. Exact transport settings and retry details were appended during generation after the pilot; scoring criteria and samples were not changed.", "conclusion": "No clear S2.1 Pro naturalness advantage in this English audition: the primary interval crosses zero, and 64/120 pairs changed preference across presentation order. Identical-audio controls tied 16/16 times, but do not erase order sensitivity on differing recordings.", "perGender": { "Male": { "characters": 20, "assessableCharacters": 20, "naturalnessMeans": { "s2-pro": 3.225, "s2.1-pro": 3.15 }, "naturalnessDelta": -0.075, "stablePairPreference": { "s2-pro": 15, "s2.1-pro": 9, "tie": 6, "inconsistent": 30 }, "note": "Descriptive gender slice; no separate winner or significance claim." }, "Female": { "characters": 20, "assessableCharacters": 20, "naturalnessMeans": { "s2-pro": 3.2833, "s2.1-pro": 3.2 }, "naturalnessDelta": -0.0833, "stablePairPreference": { "s2-pro": 12, "s2.1-pro": 11, "tie": 3, "inconsistent": 34 }, "note": "Descriptive gender slice; no separate winner or significance claim." } }, "primary": { "metric": "Naturalness", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.2542, "s2.1-pro": 3.175 }, "delta": -0.0792, "ci95": [ -0.1833, 0.025 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All 40 characters have all 6 scores per model." }, "secondary": { "intelligibility": { "metric": "intelligibility", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.9917, "s2.1-pro": 3.9875 }, "delta": -0.0042, "ci95": [ -0.0208, 0.0125 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All 40 characters have all 6 scores per model." }, "expressiveness": { "metric": "expressiveness", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.2417, "s2.1-pro": 3.2208 }, "delta": -0.0208, "ci95": [ -0.1375, 0.0917 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All 40 characters have all 6 scores per model." }, "audioQuality": { "metric": "audioQuality", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.75, "s2.1-pro": 3.6917 }, "delta": -0.0583, "ci95": [ -0.125, 0.0 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All 40 characters have all 6 scores per model." }, "textFidelity": { "metric": "textFidelity", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 4.05, "s2.1-pro": 4.0208 }, "delta": -0.0292, "ci95": [ -0.0583, -0.0042 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All 40 characters have all 6 scores per model." } }, "calibration": { "pairs": 8, "reviews": 16, "ties": 16, "nonTies": 0, "unassessable": 0, "positionPreference": { "A": 0, "B": 0, "tie": 16, "unassessable": 0 }, "meanAbsoluteScoreGap": { "naturalness": 0.0, "intelligibility": 0.0, "expressiveness": 0.0, "audioQuality": 0.0, "textFidelity": 0.0 }, "maxAbsoluteScoreGap": { "naturalness": 0, "intelligibility": 0, "expressiveness": 0, "audioQuality": 0, "textFidelity": 0 }, "bothOrdersTiePairs": 8, "note": "Identical-audio controls should tie. Non-tie decisions and score gaps are disclosed as judge inconsistency, not filtered away." }, "latency": { "method": "Direct cold WebSocket requests from this test host; no warm connection pool, not end-to-end production call latency.", "models": { "s2-pro": { "samples": 120, "medianTTFAms": 477.6, "p95TTFAms": 580.8, "meanTTFAms": 495.0, "medianAfterStartMs": 242.8, "p95AfterStartMs": 327.1, "medianOutputSeconds": 14.24, "failedAttempts": 0, "samplesRetried": 0 }, "s2.1-pro": { "samples": 120, "medianTTFAms": 506.3, "p95TTFAms": 1076.2, "meanTTFAms": 584.9, "medianAfterStartMs": 265.8, "p95AfterStartMs": 860.8, "medianOutputSeconds": 14.17, "failedAttempts": 0, "samplesRetried": 0 } }, "failedAttempts": 0, "samplesRetried": 0, "note": "Timings describe successful attempts. Failed attempts are counted separately; requests were not concurrent within a matched pair." }, "orderBias": { "disagreementBreakdown": { "oppositeWinners": 21, "tieVersusWinner": 43 }, "presentedPreference": { "A": 65, "B": 114, "tie": 61, "unassessable": 0 }, "meanPresentedANaturalnessMinusB": -0.1708, "alwaysPresentedAPairs": 4, "alwaysPresentedBPairs": 17, "note": "Presentation-order repeat judgments use the same recordings and judge. They measure reliability, not independent replication." }, "modelVersions": { "gemini-3.1-pro-preview": 256 }, "successfulJudgmentTokens": { "promptTokenCount": 414054, "candidatesTokenCount": 103435, "thoughtsTokenCount": 107210, "totalTokenCount": 624699 }, "judgeRetryEvaluations": 0, "maskedResultsSha256": "bea31cc2aa8142d315a1df3845f41c732f029f773dfc9c4d8edb9227bce30f21", "confidenceNote": "Confidence values are the mean of two uncalibrated model self-reports, not probabilities of a real-world win.", "note": "No clear S2.1 Pro naturalness advantage in this English audition: the primary interval crosses zero, and 64/120 pairs changed preference across presentation order. Identical-audio controls tied 16/16 times, but do not erase order sensitivity on differing recordings. 120 matched pairs, each judged in both orders. Stable preferences require agreement; disagreements remain inconclusive. 16/16 identical-audio reviews chose tie. One AI judge, not a human preference study; repeated orders are not independent samples." }, "characters": [ { "id": "m01", "name": "Jordan", "slug": "jordan-1a5496", "gender": "Male", "rank": 1, "portrait": "portraits/m01.webp", "language": "English", "popularity": { "callers": 29, "calls": 43 }, "pairs": [ { "id": "m01-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/m01-conversation-A.wav", "model": "s2.1-pro", "duration": 13.3718 }, "b": { "src": "audio/m01-conversation-B.wav", "model": "s2-pro", "duration": 13.6736 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers slightly more natural pacing and conversational flow. Sample A is slightly more rigid, though both are highly intelligible and accurate.", "confidence": 0.7, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.6, "rationale": "Sample B offers slightly more natural pacing and conversational flow. Sample A is slightly more rigid, though both are highly intelligible and accurate." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly intelligible with perfect text fidelity. They share similar minor issues with robotic cadence and unnatural pausing, making neither clearly superior." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Scores compressed toward the center of the scale.", "Verbosity reduced for brevity.", "Both samples exhibit slightly robotic prosody and unnatural pausing." ] } }, { "id": "m01-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/m01-emotion-A.wav", "model": "s2-pro", "duration": 11.398 }, "b": { "src": "audio/m01-emotion-B.wav", "model": "s2.1-pro", "duration": 13.0002 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples are intelligible and have good text fidelity, but both fail to convey the requested emotional transitions (quiet concern to warm relief to gentle teasing). They both sound relatively flat and robotic.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are intelligible and have good text fidelity, but both fail to convey the requested emotional transitions (quiet concern to warm relief to gentle teasing). They both sound relatively flat and robotic." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text clearly and accurately but fail to capture the requested emotional transitions, resulting in a similarly flat delivery." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples lack the requested emotional transitions.", "Both samples sound somewhat robotic and flat." ] } }, { "id": "m01-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/m01-precision-A.wav", "model": "s2.1-pro", "duration": 18.4337 }, "b": { "src": "audio/m01-precision-B.wav", "model": "s2-pro", "duration": 17.6907 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A is preferred due to slightly more natural pacing and conversational flow. Sample B feels a bit more rushed and robotic in its delivery.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A is preferred due to slightly more natural pacing and conversational flow. Sample B feels a bit more rushed and robotic in its delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text clearly and accurately. The pacing and pronunciation of numbers and names are handled well in both, with negligible differences in overall quality." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit minor robotic prosody.", "Sample A has slightly better pacing.", "Both samples exhibit slightly stiff prosody." ] } } ] }, { "id": "m02", "name": "Alex Park", "slug": "alex-park-404cbc", "gender": "Male", "rank": 2, "portrait": "portraits/m02.webp", "language": "English", "popularity": { "callers": 24, "calls": 30 }, "pairs": [ { "id": "m02-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/m02-conversation-A.wav", "model": "s2-pro", "duration": 12.7448 }, "b": { "src": "audio/m02-conversation-B.wav", "model": "s2.1-pro", "duration": 12.5126 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples deliver the text clearly and naturally with good audio quality. There is no significant difference in performance to strongly prefer one over the other.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text clearly and naturally with good audio quality. There is no significant difference in performance to strongly prefer one over the other." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a slightly more natural conversational flow and better pacing. Sample A sounds a bit more robotic and stiff in its delivery." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples are very similar in quality and delivery." ] } }, { "id": "m02-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/m02-emotion-A.wav", "model": "s2.1-pro", "duration": 12.9306 }, "b": { "src": "audio/m02-emotion-B.wav", "model": "s2-pro", "duration": 12.7216 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers slightly better pacing and flow compared to A, though both models struggle to fully capture the nuanced emotional transitions requested in the prompt.", "confidence": 0.7, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample B offers slightly better pacing and flow compared to A, though both models struggle to fully capture the nuanced emotional transitions requested in the prompt." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B provides slightly better expressiveness and naturalness, capturing the intended emotional shifts more effectively than the flatter delivery in Sample A." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples struggle to fully convey the requested emotional transitions.", "Pacing in both samples feels slightly disjointed.", "Both samples exhibit some synthetic stiffness in pacing.", "Neither sample fully captures the depth of the requested emotional transitions." ] } }, { "id": "m02-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/m02-precision-A.wav", "model": "s2-pro", "duration": 17.83 }, "b": { "src": "audio/m02-precision-B.wav", "model": "s2.1-pro", "duration": 18.9445 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A offers slightly more natural pacing and conversational flow. Sample B is also clear but feels marginally more rigid in its delivery.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample A offers slightly more natural pacing and conversational flow. Sample B is also clear but feels marginally more rigid in its delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a more natural conversational flow and better pacing. Sample A sounds slightly more stilted and deliberate." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples are highly similar in voice and delivery.", "Scores compressed toward the center as requested.", "Scores compressed toward the center of the scale." ] } } ] }, { "id": "m03", "name": "Declan Ward", "slug": "declan-ward-1fe89c", "gender": "Male", "rank": 3, "portrait": "portraits/m03.webp", "language": "English", "popularity": { "callers": 20, "calls": 26 }, "pairs": [ { "id": "m03-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/m03-conversation-A.wav", "model": "s2.1-pro", "duration": 15.4151 }, "b": { "src": "audio/m03-conversation-B.wav", "model": "s2-pro", "duration": 15.1133 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A has a slightly more natural conversational tone, whereas Sample B is overly whispery and slow. Both have minor audio quality limitations.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample A has a slightly more natural conversational tone, whereas Sample B is overly whispery and slow. Both have minor audio quality limitations." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly similar in voice, pacing, and delivery. They both convey the text accurately but share minor synthetic artifacts in their cadence, making neither clearly superior." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples have slightly muffled audio quality.", "Both samples exhibit somewhat slow and deliberate pacing.", "Both samples exhibit a slightly breathy and unnatural cadence." ] } }, { "id": "m03-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/m03-emotion-A.wav", "model": "s2.1-pro", "duration": 14.1612 }, "b": { "src": "audio/m03-emotion-B.wav", "model": "s2-pro", "duration": 14.9042 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B captures the requested emotional transitions much more naturally, whereas Sample A sounds somewhat stiff and synthetic.", "confidence": 0.825, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.85, "rationale": "Sample B captures the requested emotional transitions much more naturally, whereas Sample A sounds somewhat stiff and synthetic." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples fail to execute the requested emotional shifts. Sample A is robotic with unnatural pauses, while Sample B sounds overly distressed throughout, missing the relief and teasing entirely." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples are evaluated on a compressed scale.", "No reference audio provided.", "Both samples fail to capture the requested emotional transitions.", "Sample A suffers from robotic pacing and unnatural pauses.", "Sample B maintains an overly distressed tone throughout, missing the relief and teasing." ] } }, { "id": "m03-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/m03-precision-A.wav", "model": "s2-pro", "duration": 16.6922 }, "b": { "src": "audio/m03-precision-B.wav", "model": "s2.1-pro", "duration": 19.0374 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A offers a more natural conversational cadence. Sample B suffers from robotic prosody and stilted phrasing, particularly in the second half.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A offers a more natural conversational cadence. Sample B suffers from robotic prosody and stilted phrasing, particularly in the second half." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a more natural conversational flow and better pacing. Sample A sounds slightly stiff and robotic in its delivery." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit slightly flat delivery.", "Both samples pronounce '24' as '24th', which is an acceptable normalization.", "Evaluated based on provided audio only.", "Scores compressed toward the center of the scale as requested." ] } } ] }, { "id": "m04", "name": "A femboy waiting for you to call him goodboy", "slug": "a-femboy-waiting-for-you-to-call-him-goodboy-e8c958", "gender": "Male", "rank": 4, "portrait": "portraits/m04.webp", "language": "English", "popularity": { "callers": 18, "calls": 27 }, "pairs": [ { "id": "m04-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/m04-conversation-A.wav", "model": "s2-pro", "duration": 14.347 }, "b": { "src": "audio/m04-conversation-B.wav", "model": "s2.1-pro", "duration": 14.1845 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples are very similar in voice and delivery, but Sample B has slightly more natural pacing. Both suffer from some background hiss and relatively flat expression.", "confidence": 0.7, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.6, "rationale": "Both samples are very similar in voice and delivery, but Sample B has slightly more natural pacing. Both suffer from some background hiss and relatively flat expression." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B is slightly preferred due to marginally better naturalness and flow, although both samples suffer from robotic prosody and disjointed phrasing. Sample B's delivery feels slightly more cohesive." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit a somewhat robotic and disjointed prosody.", "Both samples have minor audio quality issues, including a slight synthetic warble." ] } }, { "id": "m04-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/m04-emotion-A.wav", "model": "s2.1-pro", "duration": 13.4182 }, "b": { "src": "audio/m04-emotion-B.wav", "model": "s2-pro", "duration": 13.5575 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Both samples suffer from robotic delivery and unnatural pauses. Sample B is marginally more fluid and captures a slightly better tone, though both are heavily flawed.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples suffer from robotic delivery and unnatural pauses. Sample B is marginally more fluid and captures a slightly better tone, though both are heavily flawed." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A preserves all target text, whereas Sample B omits the first word. Both samples are relatively flat and do not fully capture the requested emotional shifts." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit highly robotic and choppy prosody.", "Both samples lack the requested emotional transitions." ] } }, { "id": "m04-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/m04-precision-A.wav", "model": "s2.1-pro", "duration": 19.1303 }, "b": { "src": "audio/m04-precision-B.wav", "model": "s2-pro", "duration": 18.8284 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers slightly better pacing and naturalness. Both samples have similar audio quality limitations and text fidelity.", "confidence": 0.7, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.6, "rationale": "Sample B offers slightly better pacing and naturalness. Both samples have similar audio quality limitations and text fidelity." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from significant robotic artifacts and unnatural pacing. Neither successfully captures the requested conversational delivery, resulting in a tie." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit some background hiss and synthetic qualities.", "Both samples exhibit highly robotic prosody.", "Unnatural pauses disrupt the conversational flow." ] } } ] }, { "id": "m05", "name": "Damian Crowe", "slug": "damian-crowe-2647a6", "gender": "Male", "rank": 5, "portrait": "portraits/m05.webp", "language": "English", "popularity": { "callers": 17, "calls": 26 }, "pairs": [ { "id": "m05-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/m05-conversation-A.wav", "model": "s2-pro", "duration": 14.4631 }, "b": { "src": "audio/m05-conversation-B.wav", "model": "s2.1-pro", "duration": 13.186 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers a slightly more natural conversational flow and better pacing than Sample A, which sounds a bit more robotic.", "confidence": 0.7, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample B offers a slightly more natural conversational flow and better pacing than Sample A, which sounds a bit more robotic." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B offers slightly more natural pacing and conversational flow. Sample A is a bit more rigid in its delivery." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit slightly stiff pacing.", "Both samples exhibit a slightly stiff, read-aloud quality rather than fully natural conversational flow." ] } }, { "id": "m05-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/m05-emotion-A.wav", "model": "s2-pro", "duration": 13.4414 }, "b": { "src": "audio/m05-emotion-B.wav", "model": "s2.1-pro", "duration": 12.0017 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers better expressiveness, capturing the requested emotional transitions more effectively than Sample A, which sounds relatively flat and robotic.", "confidence": 0.85, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample B offers better expressiveness, capturing the requested emotional transitions more effectively than Sample A, which sounds relatively flat and robotic." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are intelligible and accurate to the text but lack the requested emotional transitions, sounding somewhat flat and robotic." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples lack deep emotional nuance, but B is closer to the prompt." ] } }, { "id": "m05-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/m05-precision-A.wav", "model": "s2.1-pro", "duration": 18.1783 }, "b": { "src": "audio/m05-precision-B.wav", "model": "s2-pro", "duration": 17.2727 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers a slightly more natural conversational flow. Sample A is clear but sounds a bit more stilted and disjointed between phrases.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers a slightly more natural conversational flow. Sample A is clear but sounds a bit more stilted and disjointed between phrases." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text accurately with clear pronunciation of numbers and names. The pacing and expressiveness are very similar, resulting in a tie." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [] } } ] }, { "id": "m06", "name": "Jacob Winchester", "slug": "jacob-winchester", "gender": "Male", "rank": 6, "portrait": "portraits/m06.webp", "language": "English", "popularity": { "callers": 13, "calls": 15 }, "pairs": [ { "id": "m06-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/m06-conversation-A.wav", "model": "s2-pro", "duration": 10.9104 }, "b": { "src": "audio/m06-conversation-B.wav", "model": "s2.1-pro", "duration": 11.2123 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A provides a much cleaner recording with better audio quality. Sample B suffers from noticeable background noise and slight distortion, making A the clear preference.", "confidence": 0.85, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample A provides a much cleaner recording with better audio quality. Sample B suffers from noticeable background noise and slight distortion, making A the clear preference." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text accurately with natural, conversational prosody and clear audio quality. There is no significant difference to prefer one over the other." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [] } }, { "id": "m06-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/m06-emotion-A.wav", "model": "s2.1-pro", "duration": 11.398 }, "b": { "src": "audio/m06-emotion-B.wav", "model": "s2-pro", "duration": 11.9321 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Both samples suffer from noticeable background noise. Sample A is preferred for slightly better expressiveness and pacing compared to B's flatter delivery.", "confidence": 0.65, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples suffer from noticeable background noise. Sample A is preferred for slightly better expressiveness and pacing compared to B's flatter delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.6, "rationale": "Both samples exhibit noticeable background noise and somewhat flat delivery. Sample B is preferred for slightly better execution of the requested emotional transitions." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 2.5 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 2.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples have noticeable background noise.", "Sample B has a very flat delivery.", "Compressed scores toward the center.", "Reduced verbosity in rationale and summaries." ] } }, { "id": "m06-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/m06-precision-A.wav", "model": "s2.1-pro", "duration": 15.4151 }, "b": { "src": "audio/m06-precision-B.wav", "model": "s2-pro", "duration": 14.138 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples deliver the text accurately with clear pronunciation and natural pacing. There are no significant defects in either recording, making them equally usable.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text accurately with clear pronunciation and natural pacing. There are no significant defects in either recording, making them equally usable." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B offers slightly more natural phrasing and conversational pacing than Sample A. Both suffer from similar minor audio quality issues." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 3.5 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit noticeable background noise or hiss.", "Both samples have a slightly synthetic cadence." ] } } ] }, { "id": "m07", "name": "Dominic Vance", "slug": "dominic-vance-39abf1", "gender": "Male", "rank": 7, "portrait": "portraits/m07.webp", "language": "English", "popularity": { "callers": 11, "calls": 19 }, "pairs": [ { "id": "m07-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/m07-conversation-A.wav", "model": "s2.1-pro", "duration": 15.3455 }, "b": { "src": "audio/m07-conversation-B.wav", "model": "s2-pro", "duration": 18.573 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A provides a more natural conversational flow. Sample B suffers from excessive and unnatural pausing that disrupts the phrasing.", "confidence": 0.9, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample A provides a more natural conversational flow. Sample B suffers from excessive and unnatural pausing that disrupts the phrasing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample B offers a much more natural conversational flow. Sample A suffers from severe robotic prosody and unnatural pauses between words, making it sound highly synthetic." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 4, "expressiveness": 2, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit some stiffness, but A is significantly more robotic." ] } }, { "id": "m07-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/m07-emotion-A.wav", "model": "s2.1-pro", "duration": 15.0436 }, "b": { "src": "audio/m07-emotion-B.wav", "model": "s2-pro", "duration": 17.7139 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A successfully captures the intended emotional transitions from concern to relief. Sample B sounds aggressive and robotic, completely missing the requested delivery.", "confidence": 0.875, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.95, "rationale": "Sample A successfully captures the intended emotional transitions from concern to relief. Sample B sounds aggressive and robotic, completely missing the requested delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B captures the intended emotional transitions and conversational pacing much better. Sample A suffers from robotic, overly dramatic prosody that ignores the requested delivery." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 }, "B": { "naturalness": 2, "intelligibility": 4, "expressiveness": 2, "audioQuality": 3.5 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "limitations": [ "Sample B completely misses the requested emotional delivery.", "Sample A has highly unnatural, robotic pacing.", "Sample B has minor audio compression artifacts." ] } }, { "id": "m07-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/m07-precision-A.wav", "model": "s2-pro", "duration": 19.7805 }, "b": { "src": "audio/m07-precision-B.wav", "model": "s2.1-pro", "duration": 19.9662 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A provides a much more natural flow and conversational cadence. Sample B suffers from severe choppiness, inserting unnatural pauses between almost every phrase.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.85, "rationale": "Sample A provides a much more natural flow and conversational cadence. Sample B suffers from severe choppiness, inserting unnatural pauses between almost every phrase." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.85, "rationale": "Sample B offers a much more natural and conversational flow. Sample A suffers from excessive, unnatural pausing that disrupts the phrasing." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 }, "B": { "naturalness": 2, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit some synthetic qualities.", "Scores are compressed toward the center of the scale as requested.", "Evaluated based on provided audio only.", "No reference voice provided." ] } } ] }, { "id": "m08", "name": "Argos", "slug": "argos-763339", "gender": "Male", "rank": 8, "portrait": "portraits/m08.webp", "language": "English", "popularity": { "callers": 11, "calls": 12 }, "pairs": [ { "id": "m08-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/m08-conversation-A.wav", "model": "s2.1-pro", "duration": 14.7185 }, "b": { "src": "audio/m08-conversation-B.wav", "model": "s2-pro", "duration": 15.6241 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples exhibit highly robotic prosody and unnatural pacing, making them sound very synthetic. Neither is clearly better than the other.", "confidence": 0.85, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.9, "rationale": "Both samples exhibit highly robotic prosody and unnatural pacing, making them sound very synthetic. Neither is clearly better than the other." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B sounds slightly more natural and conversational than Sample A, which has a more pronounced robotic cadence and disjointed phrasing." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3.5 }, "B": { "naturalness": 2, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3.5 } }, "fidelity": { "A": "5.0/5 mean text-fidelity score", "B": "5.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit a noticeable synthetic accent or robotic prosody." ] } }, { "id": "m08-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/m08-emotion-A.wav", "model": "s2.1-pro", "duration": 13.604 }, "b": { "src": "audio/m08-emotion-B.wav", "model": "s2-pro", "duration": 13.1628 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A captures the requested emotional shifts more effectively, moving from concern to relief. Sample B is flatter and exhibits a slightly more synthetic tone.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A captures the requested emotional shifts more effectively, moving from concern to relief. Sample B is flatter and exhibits a slightly more synthetic tone." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B provides a more natural delivery with better emotional transitions that align with the directions. Sample A suffers from robotic prosody and noticeable audio artifacts." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 4, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 2.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [] } }, { "id": "m08-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/m08-precision-A.wav", "model": "s2-pro", "duration": 14.9739 }, "b": { "src": "audio/m08-precision-B.wav", "model": "s2.1-pro", "duration": 15.3455 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples deliver the text accurately with a similar accent and pacing. Neither has significant defects, resulting in a tie.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text accurately with a similar accent and pacing. Neither has significant defects, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are intelligible and accurate to the text, but both feature a strong non-native accent that affects naturalness. Differences in pacing and delivery are negligible." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples feature a strong accent, but intelligibility remains acceptable.", "Both samples feature a strong non-native accent that impacts naturalness." ] } } ] }, { "id": "m09", "name": "Luca Castellano", "slug": "luca-castellano-6ec81a", "gender": "Male", "rank": 9, "portrait": "portraits/m09.webp", "language": "English", "popularity": { "callers": 10, "calls": 21 }, "pairs": [ { "id": "m09-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/m09-conversation-A.wav", "model": "s2.1-pro", "duration": 11.1426 }, "b": { "src": "audio/m09-conversation-B.wav", "model": "s2-pro", "duration": 12.2804 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples provide clear, intelligible, and natural-sounding speech that accurately reflects the target text. The differences in delivery are negligible, resulting in a tie.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples provide clear, intelligible, and natural-sounding speech that accurately reflects the target text. The differences in delivery are negligible, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B offers slightly better pacing and a more natural conversational flow. Sample A's pauses feel slightly more disjointed." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples are very similar in delivery and quality, making preference subjective." ] } }, { "id": "m09-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/m09-emotion-A.wav", "model": "s2-pro", "duration": 11.4213 }, "b": { "src": "audio/m09-emotion-B.wav", "model": "s2.1-pro", "duration": 10.8872 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples are highly intelligible and accurate to the text, but neither successfully captures the requested emotional shifts from concern to relief to teasing.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly intelligible and accurate to the text, but neither successfully captures the requested emotional shifts from concern to relief to teasing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly intelligible and accurate to the text, but both fail to capture the dynamic emotional shifts requested in the prompt, resulting in a flat delivery." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both models struggle to convey the requested emotional transitions.", "Both samples struggle to convey the requested emotional transitions." ] } }, { "id": "m09-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/m09-precision-A.wav", "model": "s2-pro", "duration": 16.3671 }, "b": { "src": "audio/m09-precision-B.wav", "model": "s2.1-pro", "duration": 15.0668 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples deliver the text clearly and accurately with natural pacing and good audio quality. There is no clear winner.", "confidence": 0.9, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.9, "rationale": "Both samples deliver the text clearly and accurately with natural pacing and good audio quality. There is no clear winner." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.9, "rationale": "Both samples deliver the text accurately with clear pronunciation and natural pacing. There is no significant difference in quality or expressiveness to distinguish a clear winner." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples are very similar in quality and delivery." ] } } ] }, { "id": "m10", "name": "I want a dark but clingy and possessive mafia boss", "slug": "i-want-a-dark-but-clingy-and-possessive-mafia-boss-b9f5a1", "gender": "Male", "rank": 10, "portrait": "portraits/m10.webp", "language": "English", "popularity": { "callers": 6, "calls": 17 }, "pairs": [ { "id": "m10-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/m10-conversation-A.wav", "model": "s2-pro", "duration": 12.2108 }, "b": { "src": "audio/m10-conversation-B.wav", "model": "s2.1-pro", "duration": 11.5142 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples exhibit similar levels of naturalness, intelligibility, and expressiveness. The pacing and delivery are nearly identical, making it a tie.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit similar levels of naturalness, intelligibility, and expressiveness. The pacing and delivery are nearly identical, making it a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B has slightly better pacing and naturalness compared to Sample A, which sounds a bit more robotic. Both have minor audio quality limitations." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit slight background hiss and synthetic cadence." ] } }, { "id": "m10-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/m10-emotion-A.wav", "model": "s2-pro", "duration": 10.9801 }, "b": { "src": "audio/m10-emotion-B.wav", "model": "s2.1-pro", "duration": 11.4445 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples are highly intelligible and accurate to the text, but both fail to deliver the requested emotional transitions, sounding relatively flat throughout.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly intelligible and accurate to the text, but both fail to deliver the requested emotional transitions, sounding relatively flat throughout." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B offers a slightly more natural cadence and a marginally better attempt at the requested emotional shifts, though both samples are relatively flat." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples fail to convey the requested emotional transitions.", "Both samples struggle to convey the requested emotional transitions.", "Scores are compressed toward the center of the scale as requested." ] } }, { "id": "m10-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/m10-precision-A.wav", "model": "s2.1-pro", "duration": 16.3207 }, "b": { "src": "audio/m10-precision-B.wav", "model": "s2-pro", "duration": 15.8098 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers a much more natural conversational flow and pacing. Sample A suffers from robotic prosody and unnatural pauses between phrases.", "confidence": 0.9, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample B offers a much more natural conversational flow and pacing. Sample A suffers from robotic prosody and unnatural pauses between phrases." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample A flows naturally and maintains a conversational cadence. Sample B suffers from excessive, unnatural pausing that disrupts the phrasing." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both models read '24' as '24th', which is an acceptable normalization." ] } } ] }, { "id": "m11", "name": "Dante Moretti", "slug": "dante-moretti-fb1fd8", "gender": "Male", "rank": 11, "portrait": "portraits/m11.webp", "language": "English", "popularity": { "callers": 6, "calls": 11 }, "pairs": [ { "id": "m11-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/m11-conversation-A.wav", "model": "s2-pro", "duration": 13.4182 }, "b": { "src": "audio/m11-conversation-B.wav", "model": "s2.1-pro", "duration": 13.186 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers slightly more natural pacing and phrasing. Sample A sounds a bit more robotic in its delivery, though both are highly intelligible and accurate to the text.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample B offers slightly more natural pacing and phrasing. Sample A sounds a bit more robotic in its delivery, though both are highly intelligible and accurate to the text." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers slightly more natural phrasing and intonation. Sample B sounds a bit more stilted and robotic in its delivery." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [] } }, { "id": "m11-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/m11-emotion-A.wav", "model": "s2-pro", "duration": 13.0931 }, "b": { "src": "audio/m11-emotion-B.wav", "model": "s2.1-pro", "duration": 12.7448 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B is slightly better because it attempts a minor sigh at the start of the second sentence, showing a tiny bit of relief. Sample A is completely flat throughout.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B is slightly better because it attempts a minor sigh at the start of the second sentence, showing a tiny bit of relief. Sample A is completely flat throughout." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly intelligible and accurate to the text, but both exhibit slightly stiff prosody and struggle to fully capture the nuanced emotional transitions requested." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples lack the requested emotional transitions.", "Both samples sound somewhat robotic and flat." ] } }, { "id": "m11-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/m11-precision-A.wav", "model": "s2-pro", "duration": 18.3176 }, "b": { "src": "audio/m11-precision-B.wav", "model": "s2.1-pro", "duration": 17.8532 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers slightly better naturalness and pacing. Sample A has a more noticeable robotic cadence and a slight pause before 'Wait'. Both handle the text and numbers well.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers slightly better naturalness and pacing. Sample A has a more noticeable robotic cadence and a slight pause before 'Wait'. Both handle the text and numbers well." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers a more natural conversational cadence and better intonation, particularly on the question. Sample B sounds slightly flat and robotic in comparison." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples mispronounce 'Maya' as 'My-uh' instead of 'Mah-yuh', though this is a common variant.", "Both samples have a slightly robotic cadence.", "Both models read 'July 24' as 'July 24th', which is an acceptable normalization." ] } } ] }, { "id": "m12", "name": "Horgan", "slug": "horgan-267181", "gender": "Male", "rank": 12, "portrait": "portraits/m12.webp", "language": "English", "popularity": { "callers": 6, "calls": 8 }, "pairs": [ { "id": "m12-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/m12-conversation-A.wav", "model": "s2-pro", "duration": 12.0482 }, "b": { "src": "audio/m12-conversation-B.wav", "model": "s2.1-pro", "duration": 11.7463 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A offers a slightly more natural conversational flow and cadence. Both samples are highly intelligible and accurate to the text.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample A offers a slightly more natural conversational flow and cadence. Both samples are highly intelligible and accurate to the text." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text accurately with clear pronunciation and natural pacing. There are no significant differences in audio quality or expressiveness, resulting in a tie." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples are very similar in voice and delivery.", "Sample B has slightly less natural phrasing.", "Both samples perform well, making differentiation difficult." ] } }, { "id": "m12-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/m12-emotion-A.wav", "model": "s2-pro", "duration": 11.9785 }, "b": { "src": "audio/m12-emotion-B.wav", "model": "s2.1-pro", "duration": 11.2355 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples are highly intelligible and accurate to the text, but neither successfully captures the requested emotional shifts from concern to relief to teasing. They are functionally equivalent.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly intelligible and accurate to the text, but neither successfully captures the requested emotional shifts from concern to relief to teasing. They are functionally equivalent." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are intelligible and accurate to the text, but neither successfully captures the requested emotional shifts from concern to relief to teasing. They both sound relatively flat and lack dynamic range." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples struggle to convey the requested emotional transitions.", "Pacing in both samples feels slightly unnatural.", "Expressiveness is generally flat in both recordings." ] } }, { "id": "m12-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/m12-precision-A.wav", "model": "s2.1-pro", "duration": 16.5297 }, "b": { "src": "audio/m12-precision-B.wav", "model": "s2-pro", "duration": 16.251 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers a more natural conversational flow and cadence. Sample A is accurate but features slightly stilted phrasing and unnatural pauses.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers a more natural conversational flow and cadence. Sample A is accurate but features slightly stilted phrasing and unnatural pauses." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample A provides a smooth, conversational delivery that fits the prompt well. Sample B suffers from excessive, unnatural pausing that disrupts the flow and sounds robotic." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Evaluated based on provided audio only.", "No reference voice provided for similarity comparison." ] } } ] }, { "id": "m13", "name": "陳灝", "slug": "love-ab00ae", "gender": "Male", "rank": 13, "portrait": "portraits/m13.webp", "language": "English", "popularity": { "callers": 6, "calls": 8 }, "pairs": [ { "id": "m13-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/m13-conversation-A.wav", "model": "s2.1-pro", "duration": 14.0219 }, "b": { "src": "audio/m13-conversation-B.wav", "model": "s2-pro", "duration": 15.299 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B is slightly preferred due to a marginally more natural cadence, although both suffer from significant robotic prosody. Sample A's pacing is slightly more disjointed.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B is slightly preferred due to a marginally more natural cadence, although both suffer from significant robotic prosody. Sample A's pacing is slightly more disjointed." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a slightly robotic cadence and flat delivery. They are equally intelligible and accurate to the text, resulting in a tie." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit robotic prosody and unnatural pacing.", "Both samples have minor audio quality issues, such as slight buzzing." ] } }, { "id": "m13-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/m13-emotion-A.wav", "model": "s2-pro", "duration": 13.4182 }, "b": { "src": "audio/m13-emotion-B.wav", "model": "s2.1-pro", "duration": 14.1845 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples deliver the text clearly but fail to capture the requested emotional transitions, resulting in a flat delivery. Neither stands out as significantly better.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text clearly but fail to capture the requested emotional transitions, resulting in a flat delivery. Neither stands out as significantly better." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly similar in their limitations, offering a flat, somewhat robotic delivery that ignores the requested emotional cues. Neither sample successfully conveys concern, relief, or teasing." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both models struggle to execute dynamic emotional transitions within a single utterance.", "Both samples fail to capture the requested emotional transitions.", "Both samples exhibit a somewhat robotic and flat delivery." ] } }, { "id": "m13-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/m13-precision-A.wav", "model": "s2-pro", "duration": 18.6659 }, "b": { "src": "audio/m13-precision-B.wav", "model": "s2.1-pro", "duration": 19.2928 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A has slightly more natural phrasing and pacing compared to Sample B, which sounds a bit more stilted. Both convey the text accurately.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample A has slightly more natural phrasing and pacing compared to Sample B, which sounds a bit more stilted. Both convey the text accurately." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B is preferred due to slightly more natural pacing and better pronunciation of the names compared to Sample A. Both samples are intelligible and faithful to the text." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit slightly robotic prosody.", "Sample A has minor pronunciation issues with names." ] } } ] }, { "id": "m14", "name": "久世 綾人", "slug": "kuze-ayato-71243c48-5e04-451f-9421-0730977af1a8", "gender": "Male", "rank": 14, "portrait": "portraits/m14.webp", "language": "English", "popularity": { "callers": 6, "calls": 6 }, "pairs": [ { "id": "m14-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/m14-conversation-A.wav", "model": "s2.1-pro", "duration": 14.8114 }, "b": { "src": "audio/m14-conversation-B.wav", "model": "s2-pro", "duration": 12.8841 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B is preferred because it maintains perfect text fidelity. Sample A omits a conjunction, altering the sentence structure slightly. Both share a similar breathy vocal quality.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B is preferred because it maintains perfect text fidelity. Sample A omits a conjunction, altering the sentence structure slightly. Both share a similar breathy vocal quality." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A is preferred because it maintains better text fidelity. Sample B omits a word from the target text. Both share a similarly breathy delivery style." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit an overly breathy delivery that sounds slightly unnatural for everyday conversation.", "Both samples feature an overly breathy, whispered delivery that feels slightly unnatural for the context." ] } }, { "id": "m14-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/m14-emotion-A.wav", "model": "s2-pro", "duration": 12.5358 }, "b": { "src": "audio/m14-emotion-B.wav", "model": "s2.1-pro", "duration": 13.5807 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A is slightly preferred due to marginally better naturalness and expressiveness, though both suffer from robotic delivery and unnatural pacing.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A is slightly preferred due to marginally better naturalness and expressiveness, though both suffer from robotic delivery and unnatural pacing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample A attempts the emotional transitions requested in the prompt, whereas Sample B remains in a whispered, worried tone throughout. Sample A's audio quality is slightly worse, but its expressiveness is more accurate." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit noticeable robotic prosody and unnatural pauses.", "Audio quality is slightly degraded in both, with some synthetic artifacts.", "Both samples struggle with natural emotional transitions.", "Sample A has noticeable background noise." ] } }, { "id": "m14-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/m14-precision-A.wav", "model": "s2-pro", "duration": 16.2046 }, "b": { "src": "audio/m14-precision-B.wav", "model": "s2.1-pro", "duration": 18.1318 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A offers slightly more natural pacing and phrasing. Sample B sounds a bit more rigid in its delivery.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample A offers slightly more natural pacing and phrasing. Sample B sounds a bit more rigid in its delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B provides clearer pronunciation and better intelligibility, particularly with names and numbers. Sample A struggles with several words, making it harder to understand." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples feature a noticeable accent, but this is not penalized.", "Prosody is slightly stiff in both.", "Both samples exhibit a noticeable accent that affects the pronunciation of certain words.", "Both samples have a somewhat robotic cadence." ] } } ] }, { "id": "m15", "name": "Rhett Callahan", "slug": "rhett-callahan-7c6b84", "gender": "Male", "rank": 15, "portrait": "portraits/m15.webp", "language": "English", "popularity": { "callers": 6, "calls": 6 }, "pairs": [ { "id": "m15-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/m15-conversation-A.wav", "model": "s2-pro", "duration": 12.977 }, "b": { "src": "audio/m15-conversation-B.wav", "model": "s2.1-pro", "duration": 13.1163 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers slightly more natural pacing and phrasing. Sample A sounds a bit more disjointed in its delivery. Both maintain good text fidelity.", "confidence": 0.65, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.6, "rationale": "Sample B offers slightly more natural pacing and phrasing. Sample A sounds a bit more disjointed in its delivery. Both maintain good text fidelity." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B provides slightly better audio clarity with less background noise compared to Sample A. Both samples deliver the text accurately with acceptable naturalness." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit minor synthetic qualities in prosody." ] } }, { "id": "m15-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/m15-emotion-A.wav", "model": "s2-pro", "duration": 13.9987 }, "b": { "src": "audio/m15-emotion-B.wav", "model": "s2.1-pro", "duration": 14.1148 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A is slightly better due to a marginally more natural cadence, though both fail to capture the requested emotional shifts. Sample B sounds more disjointed.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A is slightly better due to a marginally more natural cadence, though both fail to capture the requested emotional shifts. Sample B sounds more disjointed." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B is slightly preferred due to better audio quality and slightly more natural pacing, though both struggle with the requested emotional delivery." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 4, "expressiveness": 2, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples lack the requested emotional transitions.", "Both samples sound somewhat robotic and flat.", "Both samples sound somewhat robotic and lack natural phrasing." ] } }, { "id": "m15-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/m15-precision-A.wav", "model": "s2-pro", "duration": 18.6427 }, "b": { "src": "audio/m15-precision-B.wav", "model": "s2.1-pro", "duration": 17.5745 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B provides a more natural delivery and correctly pronounces 'Café'. Sample A mispronounces 'Café' and has a slightly more disjointed cadence.", "confidence": 0.9, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample B provides a more natural delivery and correctly pronounces 'Café'. Sample A mispronounces 'Café' and has a slightly more disjointed cadence." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample A provides a more natural delivery and correctly pronounces 'Café'. Sample B mispronounces 'Café' and has a slightly more stilted cadence." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples are evaluated on a compressed scale as requested.", "Evaluated based on provided audio only.", "Scores compressed toward the center of the scale as requested." ] } } ] }, { "id": "m16", "name": "Dean Winchester", "slug": "dean-winchester-509185", "gender": "Male", "rank": 16, "portrait": "portraits/m16.webp", "language": "English", "popularity": { "callers": 5, "calls": 9 }, "pairs": [ { "id": "m16-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/m16-conversation-A.wav", "model": "s2.1-pro", "duration": 10.3996 }, "b": { "src": "audio/m16-conversation-B.wav", "model": "s2-pro", "duration": 11.1194 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples deliver the text accurately with good audio quality and natural pacing. The differences are purely stylistic, resulting in a tie.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text accurately with good audio quality and natural pacing. The differences are purely stylistic, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B has cleaner audio quality with less background noise compared to Sample A. Both deliver the text accurately and naturally." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [] } }, { "id": "m16-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/m16-emotion-A.wav", "model": "s2.1-pro", "duration": 11.1891 }, "b": { "src": "audio/m16-emotion-B.wav", "model": "s2-pro", "duration": 10.6086 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A is preferred due to slightly cleaner audio quality. Sample B has more noticeable background noise. Both samples are somewhat flat in their delivery.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample A is preferred due to slightly cleaner audio quality. Sample B has more noticeable background noise. Both samples are somewhat flat in their delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers slightly better pacing and more natural emotional transitions compared to Sample A, which sounds a bit more rigid." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 2.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples struggle to fully capture the requested emotional transitions.", "Both samples exhibit minor synthetic artifacts.", "Emotional transitions could be more distinct." ] } }, { "id": "m16-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/m16-precision-A.wav", "model": "s2-pro", "duration": 13.6272 }, "b": { "src": "audio/m16-precision-B.wav", "model": "s2.1-pro", "duration": 12.6984 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B has slightly more natural pacing and conversational flow compared to Sample A, which feels a bit rushed in places.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B has slightly more natural pacing and conversational flow compared to Sample A, which feels a bit rushed in places." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text clearly and accurately with good naturalness and audio quality. There is no significant difference to warrant a preference." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples are very similar in quality and delivery.", "Minor pacing differences are the main distinguishing factor.", "Scores are compressed towards the center as requested." ] } } ] }, { "id": "m17", "name": "Dante Moretti", "slug": "dante-moretti-97a7e1", "gender": "Male", "rank": 17, "portrait": "portraits/m17.webp", "language": "English", "popularity": { "callers": 5, "calls": 7 }, "pairs": [ { "id": "m17-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/m17-conversation-A.wav", "model": "s2-pro", "duration": 14.0916 }, "b": { "src": "audio/m17-conversation-B.wav", "model": "s2.1-pro", "duration": 12.7912 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A offers slightly better pacing and naturalness. Sample B feels a bit more disjointed and robotic in its delivery. Both have perfect text fidelity.", "confidence": 0.7, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.6, "rationale": "Sample A offers slightly better pacing and naturalness. Sample B feels a bit more disjointed and robotic in its delivery. Both have perfect text fidelity." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly intelligible and accurate to the text. They share a similar slightly robotic cadence, making neither clearly superior." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit slightly robotic prosody." ] } }, { "id": "m17-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/m17-emotion-A.wav", "model": "s2.1-pro", "duration": 12.3036 }, "b": { "src": "audio/m17-emotion-B.wav", "model": "s2-pro", "duration": 13.0699 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B is preferred because it sounds slightly more natural and captures a bit more of the intended emotional tone than Sample A, which is very flat and robotic.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B is preferred because it sounds slightly more natural and captures a bit more of the intended emotional tone than Sample A, which is very flat and robotic." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text accurately but fail to convey the requested emotional shifts from concern to relief to teasing. They both sound relatively flat and synthetic, resulting in a tie." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples struggle to fully capture the nuanced emotional transitions requested.", "Both exhibit some robotic prosody.", "Both samples fail to capture the requested emotional transitions.", "Both samples exhibit robotic prosody." ] } }, { "id": "m17-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/m17-precision-A.wav", "model": "s2.1-pro", "duration": 18.5034 }, "b": { "src": "audio/m17-precision-B.wav", "model": "s2-pro", "duration": 17.5513 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Both samples are clear and accurate. Sample B has slightly more natural pacing and conversational flow compared to A.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples are clear and accurate. Sample B has slightly more natural pacing and conversational flow compared to A." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers a slightly more natural conversational flow and better pacing. Sample B is accurate but sounds slightly more stilted in its delivery." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit minor pacing stiffness.", "Scores compressed toward the center per instructions." ] } } ] }, { "id": "m18", "name": "Pierrot", "slug": "pierrotobbsesivekindmonsterdating-user-3440e1", "gender": "Male", "rank": 18, "portrait": "portraits/m18.webp", "language": "English", "popularity": { "callers": 5, "calls": 6 }, "pairs": [ { "id": "m18-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/m18-conversation-A.wav", "model": "s2.1-pro", "duration": 15.299 }, "b": { "src": "audio/m18-conversation-B.wav", "model": "s2-pro", "duration": 16.3207 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B has slightly more natural phrasing and less exaggerated breathiness compared to A, making it a better fit for everyday conversation.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample B has slightly more natural phrasing and less exaggerated breathiness compared to A, making it a better fit for everyday conversation." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A is preferred because it has better audio quality. Sample B contains a noticeable click/glitch artifact near the beginning, which detracts from the listening experience." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 2.5 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "limitations": [ "Both samples use an overly breathy, intimate tone that borders on unnatural for a standard everyday conversation.", "Both samples exhibit a very breathy, almost whispered delivery that feels slightly unnatural for an everyday conversation.", "Sample B has a noticeable click/glitch artifact." ] } }, { "id": "m18-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/m18-emotion-A.wav", "model": "s2.1-pro", "duration": 16.5297 }, "b": { "src": "audio/m18-emotion-B.wav", "model": "s2-pro", "duration": 15.2758 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B provides clearer audio quality and more effectively captures the requested emotional transitions. Sample A sounds slightly muffled and less dynamic in its delivery.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B provides clearer audio quality and more effectively captures the requested emotional transitions. Sample A sounds slightly muffled and less dynamic in its delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample A offers superior audio quality and better captures the requested emotional transitions. Sample B suffers from background hiss and maintains a strained, breathy tone throughout, missing the shift to warm relief." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [] } }, { "id": "m18-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/m18-precision-A.wav", "model": "s2.1-pro", "duration": 21.2665 }, "b": { "src": "audio/m18-precision-B.wav", "model": "s2-pro", "duration": 19.5018 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers more natural pacing and clearer delivery. Sample A is overly hesitant and breathy, which detracts from the conversational flow.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers more natural pacing and clearer delivery. Sample A is overly hesitant and breathy, which detracts from the conversational flow." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers a smoother, more conversational flow. Sample B suffers from awkward pauses that disrupt the natural cadence of the instructions." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit slightly unnatural pacing." ] } } ] }, { "id": "m19", "name": "Ethan", "slug": "ethan-81cb00", "gender": "Male", "rank": 19, "portrait": "portraits/m19.webp", "language": "English", "popularity": { "callers": 5, "calls": 5 }, "pairs": [ { "id": "m19-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/m19-conversation-A.wav", "model": "s2-pro", "duration": 12.3965 }, "b": { "src": "audio/m19-conversation-B.wav", "model": "s2.1-pro", "duration": 12.4894 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples are highly similar in voice, delivery, and overall quality. They both suffer from slight robotic prosody, making a tie the most appropriate outcome.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly similar in voice, delivery, and overall quality. They both suffer from slight robotic prosody, making a tie the most appropriate outcome." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a slightly more natural cadence and conversational flow compared to Sample A, which sounds a bit more hesitant." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit slightly robotic prosody and breathiness.", "Both samples exhibit a slightly muffled audio quality." ] } }, { "id": "m19-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/m19-emotion-A.wav", "model": "s2.1-pro", "duration": 12.3036 }, "b": { "src": "audio/m19-emotion-B.wav", "model": "s2-pro", "duration": 11.6303 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B provides slightly better naturalness and audio quality. Both samples fail to fully capture the dynamic emotional shifts requested, but B is more pleasant to listen to.", "confidence": 0.7, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample B provides slightly better naturalness and audio quality. Both samples fail to fully capture the dynamic emotional shifts requested, but B is more pleasant to listen to." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample A provides a slightly more natural transition between the requested emotional states, whereas Sample B relies on somewhat exaggerated breathiness." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both models struggle to execute the requested emotional transitions, remaining mostly in a whispered tone.", "Both samples exhibit minor synthetic cadence." ] } }, { "id": "m19-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/m19-precision-A.wav", "model": "s2.1-pro", "duration": 16.0652 }, "b": { "src": "audio/m19-precision-B.wav", "model": "s2-pro", "duration": 15.7402 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A offers a more natural conversational tone and better pacing. Sample B is slightly breathy and less dynamic in its delivery.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A offers a more natural conversational tone and better pacing. Sample B is slightly breathy and less dynamic in its delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample A exhibits slightly more natural pacing and conversational flow. Sample B is also clear but sounds slightly more rigid in its delivery." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples add 'th' to '24', which is an acceptable natural pronunciation." ] } } ] }, { "id": "m20", "name": "Jae-won", "slug": "jae-won-3ed8a6", "gender": "Male", "rank": 20, "portrait": "portraits/m20.webp", "language": "English", "popularity": { "callers": 5, "calls": 5 }, "pairs": [ { "id": "m20-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/m20-conversation-A.wav", "model": "s2.1-pro", "duration": 11.8625 }, "b": { "src": "audio/m20-conversation-B.wav", "model": "s2-pro", "duration": 10.7247 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A offers a more natural conversational cadence and better pacing. Sample B sounds slightly more robotic and flat in its delivery.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A offers a more natural conversational cadence and better pacing. Sample B sounds slightly more robotic and flat in its delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B is preferred because it lacks the audio glitch present in Sample A and has slightly more natural pacing, though both are somewhat robotic." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit some background noise or hiss.", "Audio quality is slightly muffled in both recordings.", "Both samples have somewhat unnatural pacing and pauses.", "Sample A has a noticeable audio glitch/click." ] } }, { "id": "m20-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/m20-emotion-A.wav", "model": "s2-pro", "duration": 9.5173 }, "b": { "src": "audio/m20-emotion-B.wav", "model": "s2.1-pro", "duration": 10.7943 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B provides a more natural and expressive delivery that better fits the requested emotional transitions. Sample A sounds somewhat robotic and lacks the warmth and relief requested in the prompt.", "confidence": 0.85, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B provides a more natural and expressive delivery that better fits the requested emotional transitions. Sample A sounds somewhat robotic and lacks the warmth and relief requested in the prompt." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample B accurately follows the target text and provides a more dynamic, expressive delivery. Sample A substitutes a word at the beginning, changing the meaning." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "limitations": [] } }, { "id": "m20-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/m20-precision-A.wav", "model": "s2.1-pro", "duration": 14.8346 }, "b": { "src": "audio/m20-precision-B.wav", "model": "s2-pro", "duration": 14.5792 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers slightly better audio clarity with less background noise. Sample A is slightly more natural in its delivery but suffers from a muffled, roomy audio quality.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample B offers slightly better audio clarity with less background noise. Sample A is slightly more natural in its delivery but suffers from a muffled, roomy audio quality." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B has slightly more natural pacing and expressiveness compared to Sample A, which sounds a bit more disjointed and robotic in its delivery." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Evaluated based on provided audio only.", "No reference voice provided for similarity check.", "Both samples have somewhat unnatural pauses.", "Both samples pronounce 'Maya' as 'My-uh', which is acceptable." ] } } ] }, { "id": "f01", "name": "Mariam", "slug": "mariam-30bd41", "gender": "Female", "rank": 1, "portrait": "portraits/f01.webp", "language": "English", "popularity": { "callers": 325, "calls": 426 }, "pairs": [ { "id": "f01-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/f01-conversation-A.wav", "model": "s2-pro", "duration": 12.6984 }, "b": { "src": "audio/f01-conversation-B.wav", "model": "s2.1-pro", "duration": 12.1179 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples deliver the text accurately with clear intelligibility and natural conversational pacing. There is no significant difference in quality.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text accurately with clear intelligibility and natural conversational pacing. There is no significant difference in quality." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.9, "rationale": "Both samples deliver the text accurately with a natural, conversational tone. There are no significant defects in either recording, making them equally usable." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4.5, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4.5, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "limitations": [] } }, { "id": "f01-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/f01-emotion-A.wav", "model": "s2-pro", "duration": 12.7216 }, "b": { "src": "audio/f01-emotion-B.wav", "model": "s2.1-pro", "duration": 11.8625 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B captures the requested emotional transitions slightly better, whereas Sample A remains a bit too flat throughout.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample B captures the requested emotional transitions slightly better, whereas Sample A remains a bit too flat throughout." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text with appropriate emotional transitions and clear articulation. There is no significant difference in quality or fidelity to warrant a preference." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples are somewhat restrained in their emotional delivery.", "Both samples are very similar in voice and delivery, making differentiation difficult." ] } }, { "id": "f01-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/f01-precision-A.wav", "model": "s2.1-pro", "duration": 16.9244 }, "b": { "src": "audio/f01-precision-B.wav", "model": "s2-pro", "duration": 17.1334 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples deliver the text clearly and naturally, handling the numbers and names accurately. There is no significant difference in quality to prefer one over the other.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text clearly and naturally, handling the numbers and names accurately. There is no significant difference in quality to prefer one over the other." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.9, "rationale": "Both samples deliver the text clearly and accurately with appropriate conversational pacing. There are no significant differences in quality, naturalness, or fidelity." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit a slightly measured pace, but it fits the instructional context." ] } } ] }, { "id": "f02", "name": "Rin", "slug": "rin-aoki", "gender": "Female", "rank": 2, "portrait": "portraits/f02.webp", "language": "English", "popularity": { "callers": 318, "calls": 412 }, "pairs": [ { "id": "f02-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/f02-conversation-A.wav", "model": "s2.1-pro", "duration": 13.5807 }, "b": { "src": "audio/f02-conversation-B.wav", "model": "s2-pro", "duration": 12.4662 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A is preferred due to more natural pacing and sentence transitions. Sample B suffers from disjointed and overly long pauses between sentences, making it sound robotic.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A is preferred due to more natural pacing and sentence transitions. Sample B suffers from disjointed and overly long pauses between sentences, making it sound robotic." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a more natural cadence and conversational tone. Sample A sounds slightly robotic in its phrasing. Both have minor audio quality limitations." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit some unnatural pacing.", "Sample A has noticeable background noise.", "Scores compressed toward the center of the scale.", "Rationale and summaries are brief." ] } }, { "id": "f02-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/f02-emotion-A.wav", "model": "s2.1-pro", "duration": 11.5606 }, "b": { "src": "audio/f02-emotion-B.wav", "model": "s2-pro", "duration": 11.6999 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A captures the requested emotional transitions slightly better than Sample B, which sounds more monotonous. Both have minor pacing issues.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A captures the requested emotional transitions slightly better than Sample B, which sounds more monotonous. Both have minor pacing issues." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample B successfully captures the requested emotional transitions, whereas Sample A is entirely flat and robotic." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit slightly unnatural pacing between sentences.", "Scores are compressed toward the center of the scale per instructions.", "Both samples exhibit some synthetic cadence.", "Sample A is completely devoid of the requested emotion." ] } }, { "id": "f02-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/f02-precision-A.wav", "model": "s2.1-pro", "duration": 19.525 }, "b": { "src": "audio/f02-precision-B.wav", "model": "s2-pro", "duration": 16.1117 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers significantly more natural phrasing and cadence. Sample A is choppy and robotic in its delivery.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample B offers significantly more natural phrasing and cadence. Sample A is choppy and robotic in its delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers smoother pacing and more natural prosody. Sample B suffers from a choppy, staccato delivery that disrupts the conversational flow." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples feature a noticeable accent, but it does not severely impact intelligibility." ] } } ] }, { "id": "f03", "name": "Antionette trella vandersloot", "slug": "antionette-trella-vandersloot-952e8b", "gender": "Female", "rank": 3, "portrait": "portraits/f03.webp", "language": "English", "popularity": { "callers": 176, "calls": 223 }, "pairs": [ { "id": "f03-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/f03-conversation-A.wav", "model": "s2-pro", "duration": 13.3253 }, "b": { "src": "audio/f03-conversation-B.wav", "model": "s2.1-pro", "duration": 13.3253 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A offers slightly better audio quality with less background noise. Both samples deliver the text accurately and naturally.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A offers slightly better audio quality with less background noise. Both samples deliver the text accurately and naturally." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples provide a natural, conversational delivery with good audio quality and accurate text fidelity. There is no clear winner as both perform equally well." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit some background noise, but B's is more pronounced.", "Both samples are very similar in quality and delivery." ] } }, { "id": "f03-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/f03-emotion-A.wav", "model": "s2-pro", "duration": 12.7216 }, "b": { "src": "audio/f03-emotion-B.wav", "model": "s2.1-pro", "duration": 13.0467 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A handles the emotional transitions slightly more naturally than Sample B. Both samples suffer from noticeable background noise, but A's delivery feels slightly more cohesive.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample A handles the emotional transitions slightly more naturally than Sample B. Both samples suffer from noticeable background noise, but A's delivery feels slightly more cohesive." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A captures the requested emotional transitions more effectively, including a natural sigh, and has slightly better audio quality. Sample B is somewhat flatter in delivery." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Compressed scores toward the center of the scale.", "Reduced verbosity in rationale and summaries." ] } }, { "id": "f03-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/f03-precision-A.wav", "model": "s2.1-pro", "duration": 17.2263 }, "b": { "src": "audio/f03-precision-B.wav", "model": "s2-pro", "duration": 17.5281 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples deliver the text clearly and accurately with appropriate conversational tone. The differences in pacing are negligible, resulting in a tie.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text clearly and accurately with appropriate conversational tone. The differences in pacing are negligible, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text clearly and accurately with good pronunciation of numbers and names. The differences in pacing and expressiveness are negligible, resulting in a tie." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples are very similar in voice and delivery.", "Minor pacing differences are subjective.", "Both samples exhibit minor pacing pauses that feel slightly unnatural." ] } } ] }, { "id": "f04", "name": "Jen", "slug": "jen-c538c2", "gender": "Female", "rank": 4, "portrait": "portraits/f04.webp", "language": "English", "popularity": { "callers": 154, "calls": 237 }, "pairs": [ { "id": "f04-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/f04-conversation-A.wav", "model": "s2.1-pro", "duration": 12.7912 }, "b": { "src": "audio/f04-conversation-B.wav", "model": "s2-pro", "duration": 12.1411 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B provides clearer audio quality and more natural phrasing. Sample A sounds slightly muffled and has less fluid transitions between sentences.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B provides clearer audio quality and more natural phrasing. Sample A sounds slightly muffled and has less fluid transitions between sentences." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers slightly more natural pacing and conversational flow. Sample B sounds a bit disjointed in its phrasing." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Evaluated based on provided audio only.", "Both samples exhibit some background noise and muffled audio quality." ] } }, { "id": "f04-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/f04-emotion-A.wav", "model": "s2-pro", "duration": 12.7216 }, "b": { "src": "audio/f04-emotion-B.wav", "model": "s2.1-pro", "duration": 14.4863 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B captures the requested emotional transitions much more effectively. Sample A sounds relatively flat and robotic, missing the nuanced delivery cues.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B captures the requested emotional transitions much more effectively. Sample A sounds relatively flat and robotic, missing the nuanced delivery cues." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A captures the requested emotional transitions slightly better. Sample B is overly flat and suffers from highly unnatural, disjointed pauses between phrases." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit some synthetic qualities in their pacing.", "Both samples exhibit unnatural pacing and disjointed pauses." ] } }, { "id": "f04-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/f04-precision-A.wav", "model": "s2.1-pro", "duration": 16.5529 }, "b": { "src": "audio/f04-precision-B.wav", "model": "s2-pro", "duration": 16.6922 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers slightly better pacing and intonation, particularly during the question, making it sound more natural than A.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers slightly better pacing and intonation, particularly during the question, making it sound more natural than A." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample A offers slightly more natural phrasing and conversational flow. Sample B sounds a bit more synthetic in its cadence, though both handle the complex text well." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit minor robotic pacing.", "Both samples exhibit minor synthetic artifacts.", "Audio quality in both has slight room for improvement." ] } } ] }, { "id": "f05", "name": "vanessa", "slug": "vanessa-4f1786", "gender": "Female", "rank": 5, "portrait": "portraits/f05.webp", "language": "English", "popularity": { "callers": 99, "calls": 123 }, "pairs": [ { "id": "f05-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/f05-conversation-A.wav", "model": "s2.1-pro", "duration": 14.7185 }, "b": { "src": "audio/f05-conversation-B.wav", "model": "s2-pro", "duration": 14.8346 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B sounds slightly more natural and conversational than Sample A, which has a more stilted delivery. Both have perfect text fidelity.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B sounds slightly more natural and conversational than Sample A, which has a more stilted delivery. Both have perfect text fidelity." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B has a slightly more natural conversational flow compared to Sample A, which sounds a bit more stilted and deliberate." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit slightly robotic prosody.", "No major defects in either sample." ] } }, { "id": "f05-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/f05-emotion-A.wav", "model": "s2.1-pro", "duration": 14.4399 }, "b": { "src": "audio/f05-emotion-B.wav", "model": "s2-pro", "duration": 14.6024 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A is preferred due to slightly better pacing and naturalness. Sample B suffers from unnatural, disjointed pauses between phrases.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A is preferred due to slightly better pacing and naturalness. Sample B suffers from unnatural, disjointed pauses between phrases." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a slightly more natural delivery with better emotional transitions, capturing the relief and teasing more effectively than Sample A, which sounds flatter." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples struggle to convey the requested emotional transitions.", "Both samples exhibit somewhat robotic prosody.", "Both samples exhibit some synthetic qualities in prosody." ] } }, { "id": "f05-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/f05-precision-A.wav", "model": "s2.1-pro", "duration": 21.0343 }, "b": { "src": "audio/f05-precision-B.wav", "model": "s2-pro", "duration": 17.0173 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B provides a much more natural and conversational delivery. Sample A suffers from robotic prosody and awkward, disjointed pacing.", "confidence": 0.9, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample B provides a much more natural and conversational delivery. Sample A suffers from robotic prosody and awkward, disjointed pacing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample A offers a more natural, conversational flow. Sample B suffers from slightly robotic pacing and unnatural pauses, making it sound less human." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples expand 'July 24' to 'July 24th' and '$28.50' to '28 dollars 50 cents', which are acceptable natural expansions.", "Both samples are evaluated on a compressed scale." ] } } ] }, { "id": "f06", "name": "Alicia a young white women with a curvy body", "slug": "alicia-a-young-white-women-with-a-curvy-body-b4f9fb", "gender": "Female", "rank": 6, "portrait": "portraits/f06.webp", "language": "English", "popularity": { "callers": 94, "calls": 131 }, "pairs": [ { "id": "f06-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/f06-conversation-A.wav", "model": "s2-pro", "duration": 11.5374 }, "b": { "src": "audio/f06-conversation-B.wav", "model": "s2.1-pro", "duration": 11.2355 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples deliver the text clearly and naturally with good audio quality. There are no significant differences to warrant a strong preference.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text clearly and naturally with good audio quality. There are no significant differences to warrant a strong preference." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a more natural and conversational delivery, whereas Sample A sounds slightly flat and robotic in its prosody." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples are very similar in quality and delivery, making a strong preference difficult." ] } }, { "id": "f06-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/f06-emotion-A.wav", "model": "s2.1-pro", "duration": 11.7928 }, "b": { "src": "audio/f06-emotion-B.wav", "model": "s2-pro", "duration": 10.5853 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A captures the requested emotional transitions more effectively, moving from quiet concern to relief with better naturalness than Sample B.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A captures the requested emotional transitions more effectively, moving from quiet concern to relief with better naturalness than Sample B." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B better captures the requested emotional transitions, moving from concern to relief and gentle teasing. Sample A remains relatively flat and rushed throughout." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit slight synthetic pacing.", "Emotional transitions are subtle in both." ] } }, { "id": "f06-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/f06-precision-A.wav", "model": "s2.1-pro", "duration": 16.1117 }, "b": { "src": "audio/f06-precision-B.wav", "model": "s2-pro", "duration": 14.556 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers a more natural conversational flow and better pacing. Sample A sounds slightly disjointed with unnatural pauses between phrases.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers a more natural conversational flow and better pacing. Sample A sounds slightly disjointed with unnatural pauses between phrases." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text clearly and accurately with appropriate conversational tone. Differences in pacing are negligible, resulting in a tie." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit minor pacing artifacts but are generally high quality." ] } } ] }, { "id": "f07", "name": "Hannah", "slug": "hannah-69dda7", "gender": "Female", "rank": 7, "portrait": "portraits/f07.webp", "language": "English", "popularity": { "callers": 87, "calls": 108 }, "pairs": [ { "id": "f07-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/f07-conversation-A.wav", "model": "s2.1-pro", "duration": 14.3005 }, "b": { "src": "audio/f07-conversation-B.wav", "model": "s2-pro", "duration": 15.0668 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers slightly better naturalness and flow compared to A. Sample A has a few awkward pauses and a slightly more robotic cadence in the middle.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers slightly better naturalness and flow compared to A. Sample A has a few awkward pauses and a slightly more robotic cadence in the middle." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample A offers slightly more natural pacing and phrasing. Sample B sounds a bit more stilted and robotic in the middle sentences." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit a slightly breathy, overly intimate tone that borders on unnatural for an everyday conversation.", "Pacing in both samples is somewhat slow and deliberate." ] } }, { "id": "f07-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/f07-emotion-A.wav", "model": "s2.1-pro", "duration": 14.0219 }, "b": { "src": "audio/f07-emotion-B.wav", "model": "s2-pro", "duration": 13.5111 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A captures the requested emotional transitions much more effectively. Sample B remains relatively flat and lacks the dynamic range needed for the delivery.", "confidence": 0.825, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.85, "rationale": "Sample A captures the requested emotional transitions much more effectively. Sample B remains relatively flat and lacks the dynamic range needed for the delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B successfully captures the requested emotional transitions, moving from concern to relief. Sample A remains flat and monotonous throughout, missing the intended delivery." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit slight synthetic artifacts in pacing." ] } }, { "id": "f07-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/f07-precision-A.wav", "model": "s2-pro", "duration": 19.5018 }, "b": { "src": "audio/f07-precision-B.wav", "model": "s2.1-pro", "duration": 18.6891 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A provides a more natural conversational flow and better pacing. Sample B is accurate but sounds slightly more mechanical in its delivery.", "confidence": 0.85, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A provides a more natural conversational flow and better pacing. Sample B is accurate but sounds slightly more mechanical in its delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.9, "rationale": "Both samples deliver the text accurately with clear pronunciation and natural pacing. The conversational tone is well-maintained in both, making it a tie." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples are highly intelligible and accurate to the text.", "Sample B exhibits slightly more mechanical pacing." ] } } ] }, { "id": "f08", "name": "Melissa \"Miel\" De Luna", "slug": "melissa-miel-de-luna-b1d93b", "gender": "Female", "rank": 8, "portrait": "portraits/f08.webp", "language": "English", "popularity": { "callers": 83, "calls": 103 }, "pairs": [ { "id": "f08-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/f08-conversation-A.wav", "model": "s2-pro", "duration": 17.5745 }, "b": { "src": "audio/f08-conversation-B.wav", "model": "s2.1-pro", "duration": 19.1767 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from highly robotic delivery and poor audio quality. Neither achieves the requested natural conversational tone, making them equally limited.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from highly robotic delivery and poor audio quality. Neither achieves the requested natural conversational tone, making them equally limited." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B provides a more natural conversational pace and tone. Sample A is slightly slow and overly breathy, which detracts from the intended everyday conversation feel." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit severe robotic prosody.", "Noticeable background noise and audio artifacts in both." ] } }, { "id": "f08-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/f08-emotion-A.wav", "model": "s2.1-pro", "duration": 16.4135 }, "b": { "src": "audio/f08-emotion-B.wav", "model": "s2-pro", "duration": 16.878 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B is preferred due to slightly better naturalness and expressiveness. Sample A sounds a bit more robotic and has a minor audio artifact.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B is preferred due to slightly better naturalness and expressiveness. Sample A sounds a bit more robotic and has a minor audio artifact." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B is preferred for its superior naturalness and audio quality, despite missing the initial worried tone. Sample A attempts the correct delivery but is too robotic to be usable." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit some robotic prosody.", "Sample A has a slight click at the beginning.", "Sample A suffers from robotic prosody and unnatural pauses.", "Sample B fails to capture the worried tone in the first sentence, sounding overly cheerful." ] } }, { "id": "f08-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/f08-precision-A.wav", "model": "s2.1-pro", "duration": 22.4507 }, "b": { "src": "audio/f08-precision-B.wav", "model": "s2-pro", "duration": 20.895 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers more natural phrasing and a smoother cadence. Sample A exhibits a slightly robotic rhythm and minor synthetic artifacts.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers more natural phrasing and a smoother cadence. Sample A exhibits a slightly robotic rhythm and minor synthetic artifacts." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample A provides a much more natural and conversational delivery. Sample B suffers from robotic prosody and unnatural pauses, making it sound less human." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples add 'th' to '24', which is a standard and acceptable number pronunciation.", "Scores compressed toward the center of the scale.", "Rationale and summaries are brief." ] } } ] }, { "id": "f09", "name": "red head female", "slug": "red-head-female-f9dab4", "gender": "Female", "rank": 9, "portrait": "portraits/f09.webp", "language": "English", "popularity": { "callers": 81, "calls": 98 }, "pairs": [ { "id": "f09-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/f09-conversation-A.wav", "model": "s2.1-pro", "duration": 15.1829 }, "b": { "src": "audio/f09-conversation-B.wav", "model": "s2-pro", "duration": 14.7417 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples are highly similar in voice, pacing, and audio quality. They both suffer from background noise and an overly breathy delivery that feels unnatural for a standard conversation.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly similar in voice, pacing, and audio quality. They both suffer from background noise and an overly breathy delivery that feels unnatural for a standard conversation." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a more natural conversational pacing. Sample A is overly breathy and slow, making it sound slightly unnatural for an everyday conversation." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit noticeable background hiss.", "Both samples use an overly breathy, ASMR-like delivery.", "Both samples exhibit noticeable background noise and hiss." ] } }, { "id": "f09-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/f09-emotion-A.wav", "model": "s2-pro", "duration": 13.8594 }, "b": { "src": "audio/f09-emotion-B.wav", "model": "s2.1-pro", "duration": 13.8129 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B provides a more natural delivery with better pacing and emotional transitions. Sample A sounds slightly disjointed and overly slow in its phrasing.", "confidence": 0.825, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.85, "rationale": "Sample B provides a more natural delivery with better pacing and emotional transitions. Sample A sounds slightly disjointed and overly slow in its phrasing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A better captures the requested emotional transitions, whereas Sample B remains uniformly whispery and lacks the requested warmth and teasing. Sample A also has slightly better audio quality." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 2.5 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit minor pacing issues, but B is more fluid.", "Scores compressed toward the center of the scale.", "Summaries and rationales reduced in verbosity." ] } }, { "id": "f09-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/f09-precision-A.wav", "model": "s2-pro", "duration": 17.0637 }, "b": { "src": "audio/f09-precision-B.wav", "model": "s2.1-pro", "duration": 17.83 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples demonstrate similar levels of robotic prosody and minor audio quality issues. Neither clearly outperforms the other in naturalness or expressiveness, making it a tie.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples demonstrate similar levels of robotic prosody and minor audio quality issues. Neither clearly outperforms the other in naturalness or expressiveness, making it a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B provides clearer audio quality and slightly better pacing. Sample A sounds muffled and lacks conversational energy." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit robotic prosody.", "Minor audio quality limitations in both recordings.", "Both samples lack strong conversational energy.", "Audio quality in A is noticeably muffled." ] } } ] }, { "id": "f10", "name": "Jami", "slug": "jami-b01a37", "gender": "Female", "rank": 10, "portrait": "portraits/f10.webp", "language": "English", "popularity": { "callers": 78, "calls": 85 }, "pairs": [ { "id": "f10-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/f10-conversation-A.wav", "model": "s2.1-pro", "duration": 11.6303 }, "b": { "src": "audio/f10-conversation-B.wav", "model": "s2-pro", "duration": 12.2108 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples have similar issues with robotic prosody and disjointed pacing between sentences. Neither stands out as significantly better.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples have similar issues with robotic prosody and disjointed pacing between sentences. Neither stands out as significantly better." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a slightly more natural conversational flow. Sample A sounds somewhat disjointed and robotic in its phrasing." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit unnatural pacing and disjointed sentence transitions." ] } }, { "id": "f10-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/f10-emotion-A.wav", "model": "s2.1-pro", "duration": 11.7463 }, "b": { "src": "audio/f10-emotion-B.wav", "model": "s2-pro", "duration": 11.4213 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B captures the requested emotional transitions much more naturally. Sample A feels slightly exaggerated and less cohesive in its pacing.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B captures the requested emotional transitions much more naturally. Sample A feels slightly exaggerated and less cohesive in its pacing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B provides a slightly better attempt at the requested emotional transitions, whereas Sample A remains relatively flat throughout." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples lack subtle emotional transitions.", "Scores are compressed towards the center as requested." ] } }, { "id": "f10-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/f10-precision-A.wav", "model": "s2-pro", "duration": 15.5776 }, "b": { "src": "audio/f10-precision-B.wav", "model": "s2.1-pro", "duration": 16.2975 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers a slightly more natural conversational flow and better pacing between sentences. Sample A sounds slightly disjointed in its transitions.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample B offers a slightly more natural conversational flow and better pacing between sentences. Sample A sounds slightly disjointed in its transitions." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a more natural conversational flow and better prosody. Sample A sounds slightly disjointed and stiff in its delivery." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples read '24' as '24th', which is a natural pronunciation.", "Both samples add 'th' to '24', which is a natural number pronunciation and acceptable." ] } } ] }, { "id": "f11", "name": "Vivienne Brooks", "slug": "vivienne-brooks-ed4cff", "gender": "Female", "rank": 11, "portrait": "portraits/f11.webp", "language": "English", "popularity": { "callers": 39, "calls": 50 }, "pairs": [ { "id": "f11-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/f11-conversation-A.wav", "model": "s2-pro", "duration": 13.5343 }, "b": { "src": "audio/f11-conversation-B.wav", "model": "s2.1-pro", "duration": 13.5807 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B is preferred due to significantly better audio quality, lacking the background hiss present in Sample A. Both samples have good text fidelity and intelligibility.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B is preferred due to significantly better audio quality, lacking the background hiss present in Sample A. Both samples have good text fidelity and intelligibility." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text accurately and clearly, but both suffer from a slightly robotic cadence. Neither sample is significantly better than the other." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Sample A has noticeable background noise and hiss.", "Sample B has slightly robotic prosody but better audio quality.", "Both samples exhibit slightly robotic prosody." ] } }, { "id": "f11-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/f11-emotion-A.wav", "model": "s2.1-pro", "duration": 13.3718 }, "b": { "src": "audio/f11-emotion-B.wav", "model": "s2-pro", "duration": 12.9073 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B captures the requested emotional transitions more effectively, whereas Sample A remains somewhat flat and robotic throughout.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B captures the requested emotional transitions more effectively, whereas Sample A remains somewhat flat and robotic throughout." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B has slightly better expressiveness and naturalness, capturing a bit more of the intended emotional transitions compared to the very flat delivery of Sample A." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples struggle to fully convey the requested emotional transitions." ] } }, { "id": "f11-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/f11-precision-A.wav", "model": "s2.1-pro", "duration": 18.4337 }, "b": { "src": "audio/f11-precision-B.wav", "model": "s2-pro", "duration": 17.8068 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers a more natural conversational flow and better pacing. Sample A sounds slightly stilted and robotic in its delivery.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers a more natural conversational flow and better pacing. Sample A sounds slightly stilted and robotic in its delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text clearly and accurately with acceptable audio quality. They both have minor pacing stiffness, making them comparable in overall quality." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Evaluated based on provided audio only.", "Scores compressed toward the center as requested.", "Both samples exhibit slightly robotic pacing.", "Scores are compressed toward the center of the scale per instructions." ] } } ] }, { "id": "f12", "name": "Nora", "slug": "nora-d25df2", "gender": "Female", "rank": 12, "portrait": "portraits/f12.webp", "language": "English", "popularity": { "callers": 39, "calls": 47 }, "pairs": [ { "id": "f12-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/f12-conversation-A.wav", "model": "s2-pro", "duration": 14.4166 }, "b": { "src": "audio/f12-conversation-B.wav", "model": "s2.1-pro", "duration": 13.9522 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers a more natural conversational cadence and better expressiveness. Sample A sounds slightly robotic and stiff in its phrasing.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers a more natural conversational cadence and better expressiveness. Sample A sounds slightly robotic and stiff in its phrasing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B offers slightly better pacing and phrasing, making it feel a bit more conversational. Both samples suffer from similar background noise issues." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit noticeable background noise or hiss.", "Both samples have slightly robotic prosody." ] } }, { "id": "f12-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/f12-emotion-A.wav", "model": "s2.1-pro", "duration": 13.9058 }, "b": { "src": "audio/f12-emotion-B.wav", "model": "s2-pro", "duration": 13.4878 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers slightly better pacing and a marginally more natural vocal tone, though both models fail to fully capture the requested emotional shifts.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample B offers slightly better pacing and a marginally more natural vocal tone, though both models fail to fully capture the requested emotional shifts." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text clearly but fail to capture the dynamic emotional shifts requested in the prompt. They both sound somewhat flat and robotic, making it a tie." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples struggle to convey the requested emotional transitions.", "Both samples exhibit some robotic prosody." ] } }, { "id": "f12-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/f12-precision-A.wav", "model": "s2-pro", "duration": 18.4105 }, "b": { "src": "audio/f12-precision-B.wav", "model": "s2.1-pro", "duration": 20.5003 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples exhibit similar robotic prosody and flat delivery. They are equally intelligible and accurate to the text, making it a tie.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit similar robotic prosody and flat delivery. They are equally intelligible and accurate to the text, making it a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers slightly better pacing and naturalness compared to Sample B, which feels a bit rushed. Both convey the text accurately." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit slightly robotic prosody.", "Evaluations are compressed toward the center of the scale as requested." ] } } ] }, { "id": "f13", "name": "Clara", "slug": "clara-a46da7", "gender": "Female", "rank": 13, "portrait": "portraits/f13.webp", "language": "English", "popularity": { "callers": 38, "calls": 53 }, "pairs": [ { "id": "f13-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/f13-conversation-A.wav", "model": "s2-pro", "duration": 13.6504 }, "b": { "src": "audio/f13-conversation-B.wav", "model": "s2.1-pro", "duration": 13.5575 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A has slightly better pacing and naturalness compared to Sample B, which sounds more disjointed and slow. Both suffer from audio quality issues.", "confidence": 0.7, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample A has slightly better pacing and naturalness compared to Sample B, which sounds more disjointed and slow. Both suffer from audio quality issues." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B is slightly preferred for having a marginally more conversational and engaging tone. Sample A sounds overly flat and detached." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit noticeable background hiss and somewhat robotic pacing.", "Both samples exhibit slightly stiff prosody.", "Neither sample sounds completely natural for a casual conversation." ] } }, { "id": "f13-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/f13-emotion-A.wav", "model": "s2.1-pro", "duration": 12.3268 }, "b": { "src": "audio/f13-emotion-B.wav", "model": "s2-pro", "duration": 12.3036 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers a slightly more natural delivery with better emotional transitions compared to Sample A, which sounds a bit flat and robotic.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers a slightly more natural delivery with better emotional transitions compared to Sample A, which sounds a bit flat and robotic." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B provides a more natural delivery with better emotional transitions that fit the requested directions. Sample A sounds slightly more monotonous and robotic." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples lack strong emotional transitions.", "Sample A sounds slightly robotic.", "Both samples are relatively short and lack complex emotional depth, but B handles the transitions better." ] } }, { "id": "f13-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/f13-precision-A.wav", "model": "s2-pro", "duration": 15.8795 }, "b": { "src": "audio/f13-precision-B.wav", "model": "s2.1-pro", "duration": 16.5529 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B has slightly better naturalness and pacing compared to Sample A. Sample A sounds a bit more rushed and robotic in its delivery.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B has slightly better naturalness and pacing compared to Sample A. Sample A sounds a bit more rushed and robotic in its delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers more natural pacing and conversational flow. Sample B feels slightly rushed during the opening sentence, making it less natural." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples have minor naturalness issues with pacing.", "Scores compressed toward the center.", "Summaries and rationales reduced in verbosity." ] } } ] }, { "id": "f14", "name": "Yumi", "slug": "yumi-94c764", "gender": "Female", "rank": 14, "portrait": "portraits/f14.webp", "language": "English", "popularity": { "callers": 34, "calls": 45 }, "pairs": [ { "id": "f14-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/f14-conversation-A.wav", "model": "s2.1-pro", "duration": 10.7015 }, "b": { "src": "audio/f14-conversation-B.wav", "model": "s2-pro", "duration": 11.2123 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples are highly intelligible and follow the text perfectly. The delivery in both is conversational, with negligible differences in quality.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly intelligible and follow the text perfectly. The delivery in both is conversational, with negligible differences in quality." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers slightly more natural phrasing and conversational flow. Sample B is also good but feels marginally more rigid in its delivery." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit slight synthetic pacing.", "Both samples are very similar in voice and delivery.", "Sample B has slightly less natural prosody." ] } }, { "id": "f14-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/f14-emotion-A.wav", "model": "s2.1-pro", "duration": 11.7463 }, "b": { "src": "audio/f14-emotion-B.wav", "model": "s2-pro", "duration": 11.1658 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B better captures the requested emotional transitions, particularly the shift from concern to relief. Sample A remains relatively flat and monotonous throughout.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B better captures the requested emotional transitions, particularly the shift from concern to relief. Sample A remains relatively flat and monotonous throughout." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B offers slightly better naturalness and pacing compared to Sample A, which sounds more disjointed between sentences. Neither sample perfectly executes the complex emotional shifts, but B is more cohesive." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit slightly unnatural prosody during emotional transitions.", "Both samples struggle to fully capture the nuanced emotional transitions requested in the prompt." ] } }, { "id": "f14-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/f14-precision-A.wav", "model": "s2-pro", "duration": 15.1365 }, "b": { "src": "audio/f14-precision-B.wav", "model": "s2.1-pro", "duration": 17.0869 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers slightly more natural pacing and conversational flow, making it a better fit for the requested delivery.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample B offers slightly more natural pacing and conversational flow, making it a better fit for the requested delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers slightly more natural pacing and conversational rhythm. Sample B feels slightly rushed in the opening sentence, though both are highly intelligible and accurate." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Evaluator cannot assess speaker likeness to a reference voice.", "Subjective preference for pacing may vary among listeners." ] } } ] }, { "id": "f15", "name": "Elena", "slug": "elena-50d35c", "gender": "Female", "rank": 15, "portrait": "portraits/f15.webp", "language": "English", "popularity": { "callers": 34, "calls": 44 }, "pairs": [ { "id": "f15-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/f15-conversation-A.wav", "model": "s2-pro", "duration": 15.09 }, "b": { "src": "audio/f15-conversation-B.wav", "model": "s2.1-pro", "duration": 13.604 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B is slightly more natural and expressive than Sample A, which sounds a bit more robotic and disjointed.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B is slightly more natural and expressive than Sample A, which sounds a bit more robotic and disjointed." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a slightly more intimate and natural conversational tone, fitting the scenario better. Sample A is clear but slightly more rigid in its delivery." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples have minor naturalness issues with pacing.", "Both samples exhibit slight pacing stiffness." ] } }, { "id": "f15-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/f15-emotion-A.wav", "model": "s2-pro", "duration": 15.0204 }, "b": { "src": "audio/f15-emotion-B.wav", "model": "s2.1-pro", "duration": 13.3485 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B successfully captures the requested emotional transitions, including a relieved tone on 'Oh'. Sample A remains uniformly breathy and monotonous throughout, failing to convey the shift in mood.", "confidence": 0.85, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample B successfully captures the requested emotional transitions, including a relieved tone on 'Oh'. Sample A remains uniformly breathy and monotonous throughout, failing to convey the shift in mood." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a more natural cadence and better captures the emotional transitions. Sample A suffers from disjointed phrasing and unnatural pauses." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [] } }, { "id": "f15-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/f15-precision-A.wav", "model": "s2-pro", "duration": 20.4074 }, "b": { "src": "audio/f15-precision-B.wav", "model": "s2.1-pro", "duration": 19.2464 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples deliver the text clearly with good pronunciation of numbers and names. The pacing is appropriate for conversational instructions, resulting in a tie.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text clearly with good pronunciation of numbers and names. The pacing is appropriate for conversational instructions, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers slightly better pacing and a more natural conversational tone. Sample A is clear but sounds a bit more rigid and robotic in its delivery." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples are highly similar in quality and delivery.", "Both samples exhibit minor synthetic pacing." ] } } ] }, { "id": "f16", "name": "girl", "slug": "girl-98d96b", "gender": "Female", "rank": 16, "portrait": "portraits/f16.webp", "language": "English", "popularity": { "callers": 32, "calls": 43 }, "pairs": [ { "id": "f16-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/f16-conversation-A.wav", "model": "s2.1-pro", "duration": 15.3687 }, "b": { "src": "audio/f16-conversation-B.wav", "model": "s2-pro", "duration": 13.8594 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples adopt an overly breathy, whispering delivery that feels unnatural for a standard conversation. They are equally intelligible and accurate to the text, making it a tie.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples adopt an overly breathy, whispering delivery that feels unnatural for a standard conversation. They are equally intelligible and accurate to the text, making it a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A is preferred due to slightly better intelligibility and naturalness. Sample B has more pronounced audio artifacts and a less stable delivery." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 2.5 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples use an overly breathy, ASMR-style whisper that does not fit the 'natural conversation' direction.", "Both samples exhibit significant audio quality issues, including background noise and distortion.", "Both samples have a somewhat unnatural, breathy delivery that feels forced." ] } }, { "id": "f16-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/f16-emotion-A.wav", "model": "s2-pro", "duration": 13.0002 }, "b": { "src": "audio/f16-emotion-B.wav", "model": "s2.1-pro", "duration": 15.5312 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A captures the requested emotional transitions well, moving from quiet concern to relief. Sample B sounds as though the speaker is crying throughout, which does not fit the intended delivery.", "confidence": 0.875, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample A captures the requested emotional transitions well, moving from quiet concern to relief. Sample B sounds as though the speaker is crying throughout, which does not fit the intended delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.85, "rationale": "Sample A successfully captures the requested emotional transitions, whereas Sample B remains largely flat and monotone throughout." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit minor background noise.", "Sample A has minor background noise.", "Sample B lacks emotional range." ] } }, { "id": "f16-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/f16-precision-A.wav", "model": "s2.1-pro", "duration": 18.7588 }, "b": { "src": "audio/f16-precision-B.wav", "model": "s2-pro", "duration": 16.0652 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A matches the requested clear conversational delivery. Sample B is whispered, resulting in a delivery mismatch.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A matches the requested clear conversational delivery. Sample B is whispered, resulting in a delivery mismatch." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample B provides a clear, conversational delivery that fits the requested direction well. Sample A is whispered and overly breathy, making it difficult to hear and failing the 'clear conversational' prompt." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 3, "expressiveness": 2, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Evaluated based on provided audio only.", "Brevity prioritized per instructions." ] } } ] }, { "id": "f17", "name": "Remy", "slug": "remy", "gender": "Female", "rank": 17, "portrait": "portraits/f17.webp", "language": "English", "popularity": { "callers": 26, "calls": 39 }, "pairs": [ { "id": "f17-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/f17-conversation-A.wav", "model": "s2-pro", "duration": 15.717 }, "b": { "src": "audio/f17-conversation-B.wav", "model": "s2.1-pro", "duration": 15.5312 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers a slightly more natural conversational flow and better phrasing than Sample A, which sounds more disjointed.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers a slightly more natural conversational flow and better phrasing than Sample A, which sounds more disjointed." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from the same robotic prosody and unnatural pauses between sentences, making them sound like concatenated phrases rather than natural speech. Neither is clearly better." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit slightly robotic prosody.", "Both samples exhibit highly unnatural, disjointed pacing with excessive pauses between sentences." ] } }, { "id": "f17-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/f17-emotion-A.wav", "model": "s2-pro", "duration": 14.6488 }, "b": { "src": "audio/f17-emotion-B.wav", "model": "s2.1-pro", "duration": 14.2077 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from severe robotic prosody and unnatural phrasing. Sample A is entirely flat, while Sample B is overly dramatic and disjointed. Neither successfully follows the delivery directions.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from severe robotic prosody and unnatural phrasing. Sample A is entirely flat, while Sample B is overly dramatic and disjointed. Neither successfully follows the delivery directions." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B better captures the requested emotional transitions, moving from quiet concern to relief. Sample A sounds overly bright and lacks the nuanced delivery requested." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit highly robotic prosody and fail to capture the nuanced emotional transitions requested." ] } }, { "id": "f17-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/f17-precision-A.wav", "model": "s2-pro", "duration": 18.3873 }, "b": { "src": "audio/f17-precision-B.wav", "model": "s2.1-pro", "duration": 20.0127 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples deliver the text clearly but suffer from slightly robotic prosody and unnatural pausing. Neither stands out as significantly better.", "confidence": 0.85, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text clearly but suffer from slightly robotic prosody and unnatural pausing. Neither stands out as significantly better." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample B offers a much more natural and conversational flow. Sample A suffers from stilted pacing and unnatural pauses between phrases." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit slightly robotic prosody and unnatural pausing.", "Scores compressed toward the center of the scale.", "Summaries and rationales reduced in verbosity." ] } } ] }, { "id": "f18", "name": "Amanda", "slug": "amanda-d62308", "gender": "Female", "rank": 18, "portrait": "portraits/f18.webp", "language": "English", "popularity": { "callers": 26, "calls": 37 }, "pairs": [ { "id": "f18-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/f18-conversation-A.wav", "model": "s2.1-pro", "duration": 14.1845 }, "b": { "src": "audio/f18-conversation-B.wav", "model": "s2-pro", "duration": 14.8114 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples have similar pacing and slightly stilted delivery with noticeable pauses between sentences. Neither stands out as significantly better.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples have similar pacing and slightly stilted delivery with noticeable pauses between sentences. Neither stands out as significantly better." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a more natural conversational flow with better pacing. Sample A suffers from slightly stilted and unnatural pauses between sentences." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit slightly unnatural pauses between sentences." ] } }, { "id": "f18-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/f18-emotion-A.wav", "model": "s2-pro", "duration": 13.3021 }, "b": { "src": "audio/f18-emotion-B.wav", "model": "s2.1-pro", "duration": 13.4414 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B captures the requested emotional transitions more effectively, sounding more natural and expressive. Sample A is somewhat flat and robotic in its delivery.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B captures the requested emotional transitions more effectively, sounding more natural and expressive. Sample A is somewhat flat and robotic in its delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B is preferred due to slightly better text fidelity and more natural pronunciation. Sample A mispronounces 'Oh' as 'Huh' or 'Ho'." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "limitations": [ "Both samples lack strong dynamic range for the emotional transitions.", "Both samples sound somewhat stiff and robotic in pacing." ] } }, { "id": "f18-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/f18-precision-A.wav", "model": "s2.1-pro", "duration": 19.2 }, "b": { "src": "audio/f18-precision-B.wav", "model": "s2-pro", "duration": 18.7588 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers a more natural conversational flow and better intonation, particularly on the question. Sample A sounds slightly robotic with awkward pacing.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers a more natural conversational flow and better intonation, particularly on the question. Sample A sounds slightly robotic with awkward pacing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A delivers the text with more natural conversational prosody. Sample B has an unnatural, slightly robotic pitch shift on the word 'Wait'." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [] } } ] }, { "id": "f19", "name": "Sahithi", "slug": "sahithi-17e3f8", "gender": "Female", "rank": 19, "portrait": "portraits/f19.webp", "language": "English", "popularity": { "callers": 24, "calls": 34 }, "pairs": [ { "id": "f19-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/f19-conversation-A.wav", "model": "s2-pro", "duration": 13.9755 }, "b": { "src": "audio/f19-conversation-B.wav", "model": "s2.1-pro", "duration": 12.3036 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples are highly similar in voice quality, pacing, and expressiveness. They both suffer from slight robotic prosody and background noise, making neither a clear winner.", "confidence": 0.7, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly similar in voice quality, pacing, and expressiveness. They both suffer from slight robotic prosody and background noise, making neither a clear winner." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.6, "rationale": "Sample B is slightly preferred due to marginally better audio clarity and a slightly more natural cadence, though both share similar limitations." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit noticeable background noise.", "Both samples have a slightly robotic prosody.", "Both samples exhibit slightly robotic prosody and muffled audio quality." ] } }, { "id": "f19-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/f19-emotion-A.wav", "model": "s2-pro", "duration": 12.8841 }, "b": { "src": "audio/f19-emotion-B.wav", "model": "s2.1-pro", "duration": 13.7897 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Both samples suffer from robotic prosody and fail to deliver the nuanced emotional shifts requested. Sample A is slightly preferred due to marginally better fluidity compared to B's disjointed delivery.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples suffer from robotic prosody and fail to deliver the nuanced emotional shifts requested. Sample A is slightly preferred due to marginally better fluidity compared to B's disjointed delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B provides more natural phrasing and better captures the requested emotional transitions. Sample A sounds more robotic and flat in its delivery." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 2, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "limitations": [ "Both samples exhibit significant robotic prosody.", "Neither sample successfully captures the requested emotional transitions.", "Both samples exhibit some synthetic artifacts." ] } }, { "id": "f19-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/f19-precision-A.wav", "model": "s2-pro", "duration": 16.0885 }, "b": { "src": "audio/f19-precision-B.wav", "model": "s2.1-pro", "duration": 22.8687 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A provides a more natural delivery with better audio quality. Sample B suffers from noticeable background noise and robotic pacing.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample A provides a more natural delivery with better audio quality. Sample B suffers from noticeable background noise and robotic pacing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B is preferred because it sounds much more natural, despite being slightly rushed. Sample A suffers from severe robotic prosody and a pronunciation error on the first name." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 2, "audioQuality": 2.5 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Sample A has extremely robotic prosody.", "Sample B is somewhat rushed." ] } } ] }, { "id": "f20", "name": "Kinsley Zimmerman.", "slug": "kinsley-zimmerman-ae675b", "gender": "Female", "rank": 20, "portrait": "portraits/f20.webp", "language": "English", "popularity": { "callers": 23, "calls": 31 }, "pairs": [ { "id": "f20-conversation", "scenario": "Everyday conversation", "scenarioId": "conversation", "text": "Hey, I saved you a seat by the window. The rain finally stopped, and the whole street smells like summer. Tell me how your day really went. We can start with the good part, or the part you wish you could do over.", "language": "English", "direction": "Natural conversation; no performance tags", "a": { "src": "audio/f20-conversation-A.wav", "model": "s2.1-pro", "duration": 14.138 }, "b": { "src": "audio/f20-conversation-B.wav", "model": "s2-pro", "duration": 13.6272 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers better audio quality and stricter text fidelity. Sample A adds an extra word and suffers from noticeable background noise.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers better audio quality and stricter text fidelity. Sample A adds an extra word and suffers from noticeable background noise." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A provides a more natural conversational delivery with perfect text fidelity. Sample B adds an extra word and sounds slightly overly breathy." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "3.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Scores compressed towards the center.", "Summaries and rationales kept brief.", "Both samples exhibit slightly slow pacing." ] } }, { "id": "f20-emotion", "scenario": "Reassurance and playful relief", "scenarioId": "emotion", "text": "[worried, speaking softly] You got quiet, and I thought I had upset you. [relieved, warm] Oh, you were just looking for your keys. Come here. Take a breath. [gently teasing] Next time, give me a hint before I invent an entire disaster in my head.", "language": "English", "direction": "Quiet concern → warm relief → gentle teasing", "a": { "src": "audio/f20-emotion-A.wav", "model": "s2-pro", "duration": 12.2108 }, "b": { "src": "audio/f20-emotion-B.wav", "model": "s2.1-pro", "duration": 13.3718 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B better captures the requested emotional transitions, moving from concern to relief more naturally than Sample A, which remains somewhat flat.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B better captures the requested emotional transitions, moving from concern to relief more naturally than Sample A, which remains somewhat flat." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A provides slightly better emotional transitions, matching the requested delivery cues more effectively than Sample B. Both have good text fidelity and intelligibility." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples are relatively short, limiting deep analysis of long-term prosody.", "Both samples exhibit minor synthetic qualities in pacing.", "Emotional transitions are somewhat subtle." ] } }, { "id": "f20-precision", "scenario": "Pacing and pronunciation", "scenarioId": "precision", "text": "Maya and Julian are meeting us at Café Aurora at 7:45 on Friday, July 24. Our table is B-12, and the tickets cost $28.50 each. Wait, did you say fifteen or fifty? Let me write that down so we get it right.", "language": "English", "direction": "Clear conversational instructions; names, time, numbers and a brief question", "a": { "src": "audio/f20-precision-A.wav", "model": "s2-pro", "duration": 17.3424 }, "b": { "src": "audio/f20-precision-B.wav", "model": "s2.1-pro", "duration": 15.3222 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A offers a more natural, conversational pacing that fits the requested delivery well. Sample B feels slightly rushed and flat in its prosody.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A offers a more natural, conversational pacing that fits the requested delivery well. Sample B feels slightly rushed and flat in its prosody." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B provides more natural phrasing and better pronunciation of the name Maya. Sample A sounds slightly robotic in comparison." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "limitations": [ "Both samples slightly alter 'July 24' to 'July 24th', which is acceptable.", "Scores compressed toward the center of the scale.", "Summaries and rationales reduced in verbosity." ] } } ] } ] }