{
  "experiment": "abcd-002",
  "split": "test",
  "selection": "Three corrected mistakes and one introduced mistake, selected after evaluation to explain individual decisions; not a representative sample. Overall results use all 500 test conversations.",
  "context_presentation": "Paraphrased excerpts; both models received the identical full conversation prefix, prior tools and results, workflow catalog, and all 30 choices.",
  "action_labels": {
    "promo-code": "Issue a promo code",
    "verify-identity": "Verify identity",
    "update-order": "Update the order",
    "enter-details": "Enter the address details",
    "record-reason": "Record the complaint reason",
    "pull-up-account": "Look up the account",
    "try-again": "Try again",
    "instructions": "Provide instructions"
  },
  "sources": {
    "final-data/inputs.jsonl": "d5b81470e9d9b7ae4c2bf2c42e7d9f06533c635cc8b1da2a8f719269ed7ae193",
    "final-data/labels.jsonl": "cefc5660bc04b5d1dc5f303d5794bbbddbc48b742b5853c24f3fd30b897bfbc1",
    "final-run/base-predictions.jsonl": "4f67514f094702be254a8d09c22927cfec9c598ca12ead4923f20b48c3209fbb",
    "final-run/candidate-predictions.jsonl": "2fe81dda18cbf1fd0603ec51e1915d7d5fd453edfb9f760c4a13e30ff82b01af"
  },
  "examples": [
    {
      "id": "abcd-test-4827-turn-15",
      "title": "Verify identity before proceeding",
      "context_summary": "The agent asks for account and order IDs to verify the customer’s identity. The customer provides both.",
      "expected_action": "verify-identity",
      "unchanged": {
        "choice": "promo-code",
        "probabilities": {
          "ask-the-oracle": 0.019225532741962335,
          "enter-details": 0.002122978014350233,
          "instructions": 0.0017719343943036706,
          "log-out-in": 0.0007876366905649353,
          "make-password": 0.0005888184561175684,
          "make-purchase": 0.00033942814653257326,
          "membership": 0.0010214440097050543,
          "notify-team": 0.002122978014350233,
          "offer-refund": 0.002122978014350233,
          "promo-code": 0.7291515900466462,
          "pull-up-account": 0.0006318771273998527,
          "record-reason": 0.001587088066688104,
          "search-boots": 0.0003040193658414563,
          "search-faq": 0.001587088066688104,
          "search-jacket": 0.0017031474639657063,
          "search-jeans": 0.0003262514759966483,
          "search-membership": 0.0001145538201301965,
          "search-policy": 0.0003916985686489564,
          "search-pricing": 0.000454038178713544,
          "search-shirt": 0.00014532220008166833,
          "search-timing": 0.0001044220153978032,
          "select-faq": 0.00020997424725361703,
          "send-link": 0.0005451196489690843,
          "shipping-status": 0.00035083697170654253,
          "subscription-status": 0.0002022424265684397,
          "try-again": 0.00016224770058146734,
          "update-account": 0.000454038178713544,
          "update-order": 0.001901511038543283,
          "validate-purchase": 0.08989427347542292,
          "verify-identity": 0.13967493143380608
        }
      },
      "trained": {
        "choice": "verify-identity",
        "probabilities": {
          "ask-the-oracle": 0.0007514177493784288,
          "enter-details": 0.0011480801791717093,
          "instructions": 0.0017838531965414395,
          "log-out-in": 0.0016882462179220465,
          "make-password": 0.0007514177493784288,
          "make-purchase": 0.0005399350773982368,
          "membership": 0.0007514177493784288,
          "notify-team": 0.0022235781385790535,
          "offer-refund": 0.0005399350773982368,
          "promo-code": 0.0029286598614559753,
          "pull-up-account": 0.0023495011607289115,
          "record-reason": 0.000838934509090864,
          "search-boots": 0.002104403715761579,
          "search-faq": 0.0013543864628437674,
          "search-jacket": 0.0003475002109871615,
          "search-jeans": 0.0013647853538273115,
          "search-membership": 0.0018848745191341401,
          "search-policy": 0.0016100309430629998,
          "search-pricing": 0.0015121304860320588,
          "search-shirt": 0.0013647853538273115,
          "search-timing": 0.001442074485000415,
          "select-faq": 0.0017012083642282754,
          "send-link": 0.0012224122238298216,
          "shipping-status": 0.0015121304860320588,
          "subscription-status": 0.001442074485000415,
          "try-again": 0.0017838531965414395,
          "update-account": 0.0024825558627439774,
          "update-order": 0.002623145224582255,
          "validate-purchase": 0.005993382450208151,
          "verify-identity": 0.9519592895099351
        }
      }
    },
    {
      "id": "abcd-test-7423-turn-21",
      "title": "Collect the return address",
      "context_summary": "The agent approves a return and asks for the address needed for a shipping label. The customer provides it.",
      "expected_action": "enter-details",
      "unchanged": {
        "choice": "update-order",
        "probabilities": {
          "ask-the-oracle": 0.000496661299001528,
          "enter-details": 0.1321128308422657,
          "instructions": 0.00025643696711341086,
          "log-out-in": 0.0007449934333585941,
          "make-password": 0.0003846557465426225,
          "make-purchase": 7.632508611696809e-05,
          "membership": 0.003889128027082824,
          "notify-team": 0.001610937745564092,
          "offer-refund": 0.009389125784716248,
          "promo-code": 0.00018426410065174338,
          "pull-up-account": 0.00022968572725578517,
          "record-reason": 0.011703571126173568,
          "search-boots": 0.00025643696711341086,
          "search-faq": 0.00025643696711341086,
          "search-jacket": 0.0010367936281731774,
          "search-jeans": 0.0005353188562624084,
          "search-membership": 0.00011448785260557856,
          "search-policy": 0.0007600075870794055,
          "search-pricing": 0.0006807243571327707,
          "search-shirt": 0.0011808763727678274,
          "search-timing": 0.00017174658117182471,
          "select-faq": 0.0014719654807094403,
          "send-link": 0.00012097140736395838,
          "shipping-status": 0.0010367936281731774,
          "subscription-status": 0.00035147230886156206,
          "try-again": 0.00022620627260268889,
          "update-account": 0.04390144426658206,
          "update-order": 0.7700003777553708,
          "validate-purchase": 0.008409661912536752,
          "verify-identity": 0.008409661912536752
        }
      },
      "trained": {
        "choice": "enter-details",
        "probabilities": {
          "ask-the-oracle": 0.001646802224272248,
          "enter-details": 0.9382117848367301,
          "instructions": 0.002585263511705725,
          "log-out-in": 0.0018576545528926965,
          "make-password": 0.0018576545528926965,
          "make-purchase": 0.00216899245246946,
          "membership": 0.002315571497393638,
          "notify-team": 0.001663865721236641,
          "offer-refund": 0.0019427252529485327,
          "promo-code": 0.0015585407388201244,
          "pull-up-account": 0.0022918243660540456,
          "record-reason": 0.002585263511705725,
          "search-boots": 0.0022918243660540456,
          "search-faq": 0.0019427252529485327,
          "search-jacket": 0.0018576545528926965,
          "search-jeans": 7.074388111728154e-05,
          "search-membership": 7.074388111728154e-05,
          "search-policy": 5.9967854939353406e-05,
          "search-pricing": 0.001410419975855308,
          "search-shirt": 7.898335081276816e-05,
          "search-timing": 5.675383727523998e-05,
          "select-faq": 7.074388111728154e-05,
          "send-link": 0.001410419975855308,
          "shipping-status": 0.0017580921351616618,
          "subscription-status": 8.345624132066681e-05,
          "try-again": 0.0018576545528926965,
          "update-account": 0.009177859240228568,
          "update-order": 0.012772658154422346,
          "validate-purchase": 0.002585263511705725,
          "verify-identity": 0.0017580921351616618
        }
      }
    },
    {
      "id": "abcd-test-10571-turn-7",
      "title": "Find the customer’s account",
      "context_summary": "A customer complains about an out-of-stock jacket. The agent offers to look up their account and collects their name.",
      "expected_action": "pull-up-account",
      "unchanged": {
        "choice": "record-reason",
        "probabilities": {
          "ask-the-oracle": 0.01866908898585802,
          "enter-details": 0.0016664763004561342,
          "instructions": 0.0006182712668104274,
          "log-out-in": 0.0016664763004561342,
          "make-password": 0.0005537738594573658,
          "make-purchase": 0.023271072797070295,
          "membership": 0.0007706777359956795,
          "notify-team": 0.08729245639035237,
          "offer-refund": 0.0035762166247519053,
          "promo-code": 0.2107413539790654,
          "pull-up-account": 0.08729245639035237,
          "record-reason": 0.4081590861071883,
          "search-boots": 0.0005537738594573658,
          "search-faq": 0.0025893196462679255,
          "search-jacket": 0.04507097906790884,
          "search-jeans": 0.0005495798528372864,
          "search-membership": 0.00031680884236342507,
          "search-policy": 0.0006210218659733512,
          "search-pricing": 5.4356419334152796e-05,
          "search-shirt": 0.00020636700171786605,
          "search-timing": 0.00020636700171786605,
          "select-faq": 0.00031680884236342507,
          "send-link": 0.0004982113946605518,
          "shipping-status": 0.00022764462042254506,
          "subscription-status": 0.0006933515606257804,
          "try-again": 7.656321853266894e-05,
          "update-account": 0.00044089696253251157,
          "update-order": 0.0008642647218750423,
          "validate-purchase": 0.004976954462740348,
          "verify-identity": 0.09745932392085459
        }
      },
      "trained": {
        "choice": "pull-up-account",
        "probabilities": {
          "ask-the-oracle": 0.0015848111515470826,
          "enter-details": 0.001769392251818026,
          "instructions": 0.0015848111515470826,
          "log-out-in": 0.0022055526782936673,
          "make-password": 0.001599370520302483,
          "make-purchase": 0.0022055526782936673,
          "membership": 0.0022055526782936673,
          "notify-team": 0.002225814537957977,
          "offer-refund": 0.0019936197569107297,
          "promo-code": 0.0024850528779008785,
          "pull-up-account": 0.9529167026741339,
          "record-reason": 0.004812999060872793,
          "search-boots": 0.002462430919772729,
          "search-faq": 0.002774483952948399,
          "search-jacket": 0.001785647330583191,
          "search-jeans": 4.361188449799619e-05,
          "search-membership": 4.6081664692294506e-05,
          "search-policy": 0.001785647330583191,
          "search-pricing": 0.0010876417658815325,
          "search-shirt": 4.361188449799619e-05,
          "search-timing": 4.361188449799619e-05,
          "select-faq": 0.0010876417658815325,
          "send-link": 3.9062337454247854e-05,
          "shipping-status": 0.0012830862827852105,
          "subscription-status": 4.361188449799619e-05,
          "try-again": 4.6081664692294506e-05,
          "update-account": 0.0014325260907754555,
          "update-order": 0.001599370520302483,
          "validate-purchase": 0.0019936197569107297,
          "verify-identity": 0.004812999060872793
        }
      }
    },
    {
      "id": "abcd-test-9025-turn-9",
      "title": "A mistake training introduced",
      "context_summary": "The shopping cart still fails after logging out and back in. The agent asks the customer to refresh the page and try adding the item again.",
      "expected_action": "try-again",
      "unchanged": {
        "choice": "try-again",
        "probabilities": {
          "ask-the-oracle": 0.009199083602517512,
          "enter-details": 0.005485478123753277,
          "instructions": 0.27989215718378546,
          "log-out-in": 0.030907040835066766,
          "make-password": 0.0025368082538493935,
          "make-purchase": 0.16134577426225866,
          "membership": 0.0025368082538493935,
          "notify-team": 0.027682848002697262,
          "offer-refund": 0.0012662075635236887,
          "promo-code": 0.0008429880835236407,
          "pull-up-account": 0.004400694080859294,
          "record-reason": 0.00394161437876379,
          "search-boots": 0.0006057328893684978,
          "search-faq": 0.005920495741197883,
          "search-jacket": 0.0014136814195920527,
          "search-jeans": 0.0015783314410855055,
          "search-membership": 0.0005855690507869407,
          "search-policy": 0.002538558194140045,
          "search-pricing": 0.0004697695224766926,
          "search-shirt": 0.0016338066539330178,
          "search-timing": 0.0009098391212946352,
          "select-faq": 0.0014136814195920527,
          "send-link": 0.0008149255534580569,
          "shipping-status": 0.002538558194140045,
          "subscription-status": 0.0016338066539330178,
          "try-again": 0.4348877806988189,
          "update-account": 0.0016338066539330178,
          "update-order": 0.0006061513083357049,
          "validate-purchase": 0.008239444665325746,
          "verify-identity": 0.002538558194140045
        }
      },
      "trained": {
        "choice": "instructions",
        "probabilities": {
          "ask-the-oracle": 0.004227368256024069,
          "enter-details": 0.0034511528820507092,
          "instructions": 0.8369696814631967,
          "log-out-in": 0.005883150936340476,
          "make-password": 0.005883150936340476,
          "make-purchase": 0.005883150936340476,
          "membership": 0.005883150936340476,
          "notify-team": 0.005883150936340476,
          "offer-refund": 0.0024798437080791893,
          "promo-code": 0.0021021049042763465,
          "pull-up-account": 0.003091131915586118,
          "record-reason": 0.003091131915586118,
          "search-boots": 0.0024798437080791893,
          "search-faq": 0.006568355048952191,
          "search-jacket": 0.0034511528820507092,
          "search-jeans": 0.0003125139583017354,
          "search-membership": 0.004719725281565725,
          "search-policy": 0.0037863736725576263,
          "search-pricing": 0.004719725281565725,
          "search-shirt": 0.0003489121258393487,
          "search-timing": 0.0037863736725576263,
          "select-faq": 0.004719725281565725,
          "send-link": 0.005883150936340476,
          "shipping-status": 0.00027991284037978437,
          "subscription-status": 0.006568355048952191,
          "try-again": 0.06641046022661648,
          "update-account": 0.0002649106691125528,
          "update-order": 0.00027991284037978437,
          "validate-purchase": 0.00027991284037978437,
          "verify-identity": 0.0003125139583017354
        }
      }
    }
  ]
}