{"learning":{"breakdowns":{"family":{"filter":{"ddp-four":223,"diloco-four":152,"documents":256,"seed":2,"single":206},"lookup":{"ddp-four":224,"diloco-four":256,"documents":256,"seed":102,"single":247},"sort":{"ddp-four":166,"diloco-four":149,"documents":256,"seed":1,"single":188},"total":{"ddp-four":0,"diloco-four":0,"documents":256,"seed":0,"single":0}},"variant":{"0":{"ddp-four":163,"diloco-four":154,"documents":256,"seed":35,"single":163},"1":{"ddp-four":164,"diloco-four":153,"documents":256,"seed":26,"single":170},"2":{"ddp-four":136,"diloco-four":104,"documents":256,"seed":14,"single":145},"3":{"ddp-four":150,"diloco-four":146,"documents":256,"seed":30,"single":163}}},"comparisons_to_seed":{"ddp-four":{"accuracy_gain":0.49609375,"baseline_correct":105,"candidate_correct":613,"documents":1024,"exact_one_sided_p":3.587222182862047e-145,"gained":513,"interval_scope":"Approximate paired normal interval on this generated distribution, one training run","lost":5,"normal_99pct_lower":0.4550490253071793,"normal_99pct_upper":0.5371384746928207,"retention":{"documents":128,"improved":68,"mean_change":-0.002884530996095499,"normal_99pct_upper":0.005859022638954832,"scope":"descriptive development comparison; no serving approval","standard_error":0.003394236659569228,"worsened":60},"standard_error":0.01593456702906149},"diloco-four":{"accuracy_gain":0.44140625,"baseline_correct":105,"candidate_correct":557,"documents":1024,"exact_one_sided_p":5.5998602942413436e-133,"gained":454,"interval_scope":"Approximate paired normal interval on this generated distribution, one training run","lost":2,"normal_99pct_lower":0.40110117432103154,"normal_99pct_upper":0.48171132567896846,"retention":{"documents":128,"improved":14,"mean_change":0.08329359739767073,"normal_99pct_upper":0.10814018556446862,"scope":"descriptive development comparison; no serving approval","standard_error":0.009645414661024027,"worsened":114},"standard_error":0.015647417172961475},"single":{"accuracy_gain":0.5234375,"baseline_correct":105,"candidate_correct":641,"documents":1024,"exact_one_sided_p":4.0585074162004344e-158,"gained":538,"interval_scope":"Approximate paired normal interval on this generated distribution, one training run","lost":2,"normal_99pct_lower":0.48290105491892904,"normal_99pct_upper":0.563973945081071,"retention":{"documents":128,"improved":67,"mean_change":-0.0024281699544927704,"normal_99pct_upper":0.006410540564355204,"scope":"descriptive development comparison; no serving approval","standard_error":0.003431176443652164,"worsened":61},"standard_error":0.015737240439504696}},"conditions":{"maximum_one_sided_p_vs_seed":true,"minimum_accuracy_gain_vs_seed":true,"paired_99pct_lower_vs_ddp_min":false,"retention_99pct_upper_max":false},"local_vs_ddp":{"accuracy_gain":-0.0546875,"baseline_correct":613,"candidate_correct":557,"documents":1024,"exact_one_sided_p":0.9999677608096215,"gained":75,"interval_scope":"Approximate paired normal interval on this generated distribution, one training run","lost":131,"normal_99pct_lower":-0.09053925644167371,"normal_99pct_upper":-0.018835743558326283,"standard_error":0.013918529613852223},"local_vs_single_descriptive":{"accuracy_gain":-0.08203125,"baseline_correct":641,"candidate_correct":557,"documents":1024,"exact_one_sided_p":0.9999999999902098,"gained":42,"interval_scope":"Approximate paired normal interval on this generated distribution, one training run","lost":126,"normal_99pct_lower":-0.1139752789547282,"normal_99pct_upper":-0.05008722104527181,"standard_error":0.012401454130021996},"prepared":"307be733a2ed647db1bdded7443df6569805443c4ad9e02ccb4dfb491b8f5a6e","primary_arm":"diloco-four","quality_contract_passed":false,"scope":"One run on four generated task families and previously exposed public retention probes; no broad assistant or serving approval","selection":"80276cb178d7950cf514d456fb952edd95b5d9cf4ba236c0ee812d3d794f7971","serving_approved":false,"tokens_issued":0},"recovery":{"all_rank_state_and_trajectory_comparisons_passed":true,"failed_group_seconds":179.62228417396545,"killed_rank":1,"last_observed_step_before_kill":75,"rank_comparisons":[{"binding":"fd4c70b1807078ef6406f75ee84e7db69d1c58380964132c83e1b3b92de484c2","comparisons":{"model":{"scalars":0,"tensor_elements":1711376384,"tensors":218},"optimizer.pt":{"scalars":242,"tensor_elements":3422758058,"tensors":656},"outer.pt":{"scalars":222,"tensor_elements":1711376384,"tensors":218}},"contents_exactly_equal":true,"file_hashes_equal":true,"outer_digest":"bf60049ea84b7edcaa78e33e19cc397e3202b907ff71e213f91da46e66212cf8","parameter_digest":"246eea9e0b460233a7ede8c16b0f8da6ae2a8572a4c16b210dd1926788284b89","prepared":"307be733a2ed647db1bdded7443df6569805443c4ad9e02ccb4dfb491b8f5a6e","rank":0,"recovered_receipt":"a7657baba1ae2c7ce686a20f3422faf01c747a9275a40342ad4e00ee8a7689d2","resume_step":64,"scope":"Same operated GPU group and rank identities; exact tensor/scalar/RNG contents after restoring the common step-64 checkpoint","seconds":220.4084906578064,"uninterrupted_receipt":"7e8eb4560ca02ea8d66ae80471e41c4679e85e683ebbf1e2cac766aabed4937f"},{"binding":"8d1ae5c5eb8dd638b57e52b9aee1a8accb74016c1a2438410867e379c3f18170","comparisons":{"model":{"scalars":0,"tensor_elements":1711376384,"tensors":218},"optimizer.pt":{"scalars":242,"tensor_elements":3422758058,"tensors":656},"outer.pt":{"scalars":222,"tensor_elements":1711376384,"tensors":218}},"contents_exactly_equal":true,"file_hashes_equal":true,"outer_digest":"bf60049ea84b7edcaa78e33e19cc397e3202b907ff71e213f91da46e66212cf8","parameter_digest":"246eea9e0b460233a7ede8c16b0f8da6ae2a8572a4c16b210dd1926788284b89","prepared":"307be733a2ed647db1bdded7443df6569805443c4ad9e02ccb4dfb491b8f5a6e","rank":1,"recovered_receipt":"a1d6bd397abf4bbf51db5e7f7702f3634214bbc71b4321236e84f4f98a71b065","resume_step":64,"scope":"Same operated GPU group and rank identities; exact tensor/scalar/RNG contents after restoring the common step-64 checkpoint","seconds":207.3531219959259,"uninterrupted_receipt":"6fa8791a720cfb6d46b9ec90ef0f9288358c64920b8309ebe87f012f6ece448a"},{"binding":"7891d5bf1ef4b4c0ce7b6113e66d422c7033d46c204f5133748cd1939e5c1a7c","comparisons":{"model":{"scalars":0,"tensor_elements":1711376384,"tensors":218},"optimizer.pt":{"scalars":242,"tensor_elements":3422758058,"tensors":656},"outer.pt":{"scalars":222,"tensor_elements":1711376384,"tensors":218}},"contents_exactly_equal":true,"file_hashes_equal":true,"outer_digest":"bf60049ea84b7edcaa78e33e19cc397e3202b907ff71e213f91da46e66212cf8","parameter_digest":"246eea9e0b460233a7ede8c16b0f8da6ae2a8572a4c16b210dd1926788284b89","prepared":"307be733a2ed647db1bdded7443df6569805443c4ad9e02ccb4dfb491b8f5a6e","rank":2,"recovered_receipt":"00e2124feb7c839473ac385827939478df5d5fc56ea66c2b88ec7a34fd154f29","resume_step":64,"scope":"Same operated GPU group and rank identities; exact tensor/scalar/RNG contents after restoring the common step-64 checkpoint","seconds":199.7953281402588,"uninterrupted_receipt":"62ea9b6f647bbc1b67733d67ccb4551ab49a13197a160fea0dd56d49975eb300"},{"binding":"6235842e619e699ac6ef8c542667e23b04aae08fe2139a0eccae81cc5ae5284e","comparisons":{"model":{"scalars":0,"tensor_elements":1711376384,"tensors":218},"optimizer.pt":{"scalars":242,"tensor_elements":3422758058,"tensors":656},"outer.pt":{"scalars":222,"tensor_elements":1711376384,"tensors":218}},"contents_exactly_equal":true,"file_hashes_equal":true,"outer_digest":"bf60049ea84b7edcaa78e33e19cc397e3202b907ff71e213f91da46e66212cf8","parameter_digest":"246eea9e0b460233a7ede8c16b0f8da6ae2a8572a4c16b210dd1926788284b89","prepared":"307be733a2ed647db1bdded7443df6569805443c4ad9e02ccb4dfb491b8f5a6e","rank":3,"recovered_receipt":"dbce27829a017534acae2d1a29fc888e4f37f6a0d7622569ea3da97c2c639c54","resume_step":64,"scope":"Same operated GPU group and rank identities; exact tensor/scalar/RNG contents after restoring the common step-64 checkpoint","seconds":202.16933393478394,"uninterrupted_receipt":"66e146c8087030221c56aa9a211c60e1c6774e57c8d8e0b7a1ac5173b6c106bf"}],"restarted_group_seconds":357.0407500267029,"restore_step":64,"scope":"Consistency of retained host-side exact tensor/scalar/RNG comparisons under one operator; no elastic membership, Byzantine recovery or cryptographic execution proof"},"serving":{"arm":"diloco-four","conditions":{"all_requests_completed":true,"identical_successful_output_tokens":true,"no_cache_hits":true},"failure_retries":16,"four_throughput_ratio":3.00021800399805,"parameter_digest":"246eea9e0b460233a7ede8c16b0f8da6ae2a8572a4c16b210dd1926788284b89","phases":{"failure":{"concurrency":4,"endpoints":["http://172.31.30.222:9090","http://172.31.24.120:9090","http://172.31.25.24:9090","http://172.31.38.142:9090"],"median_latency_seconds":0.450490643735975,"model_digest":"246eea9e0b460233a7ede8c16b0f8da6ae2a8572a4c16b210dd1926788284b89","p95_latency_seconds":1.0279517583549023,"requests":64,"requests_per_second":6.71276497645627,"run_id":"windows-failure","scope":"Fixed committed tasks, trusted operated replicas; no payments or permissionless admission","seconds":9.534074293449521,"successful":64,"unavailable":[{"endpoint":"http://172.31.24.120:9090","error":"URLError"}]},"four":{"concurrency":4,"endpoints":["http://172.31.30.222:9090","http://172.31.24.120:9090","http://172.31.25.24:9090","http://172.31.38.142:9090"],"median_latency_seconds":0.32129482040181756,"model_digest":"246eea9e0b460233a7ede8c16b0f8da6ae2a8572a4c16b210dd1926788284b89","p95_latency_seconds":0.8844614299014211,"requests":64,"requests_per_second":9.024988958987134,"run_id":"windows-four","scope":"Fixed committed tasks, trusted operated replicas; no payments or permissionless admission","seconds":7.091421417891979,"successful":64,"unavailable":[]},"single":{"concurrency":4,"endpoints":["http://172.31.30.222:9090"],"median_latency_seconds":1.285121698398143,"model_digest":"246eea9e0b460233a7ede8c16b0f8da6ae2a8572a4c16b210dd1926788284b89","p95_latency_seconds":1.5208659349009395,"requests":64,"requests_per_second":3.0081110595831886,"run_id":"windows-single","scope":"Fixed committed tasks, trusted operated replicas; no payments or permissionless admission","seconds":21.275810211896896,"successful":64,"unavailable":[]}},"scope":"Fixed development tasks, equal client concurrency, one operator; no native payments or general chat approval","serving_fixture_passed":true},"training":{"arms":{"ddp-four":{"allocated_gpu_loop_seconds":8193.202094188,"max_rank_active_seconds":1979.2566112570019,"max_rank_loop_seconds":2048.371095644,"parameter_digest":"cd138694e67672b087d032da6c597e566e8ca40a7234085f73ce9c09353ff380","ranks":[{"active_seconds":1979.1783790579998,"checkpoint_seconds":68.95799706099979,"interface_bytes":{"docker0":{"rx":0,"tx":0},"enp39s0":{"rx":1324173268390,"tx":1338124407094}},"loop_seconds":2048.296346626,"rank":0,"sent_bytes":1338124407094,"synchronization_seconds":0,"update_seconds":1979.1783790579998},{"active_seconds":1979.1605624070032,"checkpoint_seconds":68.95744946600007,"interface_bytes":{"docker0":{"rx":0,"tx":0},"enp39s0":{"rx":1330092384573,"tx":1332992150041}},"loop_seconds":2048.2814102439997,"rank":1,"sent_bytes":1332992150041,"synchronization_seconds":0,"update_seconds":1979.1605624070032},{"active_seconds":1979.1349060939988,"checkpoint_seconds":68.95873936099997,"interface_bytes":{"docker0":{"rx":0,"tx":0},"enp39s0":{"rx":1330210341289,"tx":1332840206885}},"loop_seconds":2048.253241674,"rank":2,"sent_bytes":1332840206885,"synchronization_seconds":0,"update_seconds":1979.1349060939988},{"active_seconds":1979.2566112570019,"checkpoint_seconds":68.95801798400043,"interface_bytes":{"docker0":{"rx":0,"tx":0},"enp39s0":{"rx":1330337346074,"tx":1326705952071}},"loop_seconds":2048.371095644,"rank":3,"sent_bytes":1326705952071,"synchronization_seconds":0,"update_seconds":1979.2566112570019}],"total_sent_bytes":5330662716091,"world":4},"diloco-four":{"allocated_gpu_loop_seconds":1978.0153516120004,"max_rank_active_seconds":250.08395944700032,"max_rank_loop_seconds":494.50500539600034,"parameter_digest":"246eea9e0b460233a7ede8c16b0f8da6ae2a8572a4c16b210dd1926788284b89","ranks":[{"active_seconds":250.08395944700032,"checkpoint_seconds":244.30188048299988,"interface_bytes":{"docker0":{"rx":0,"tx":0},"enp39s0":{"rx":82782060502,"tx":82955154183}},"loop_seconds":494.50500539600034,"rank":0,"sent_bytes":82955154183,"synchronization_seconds":180.88651975499988,"update_seconds":69.19743969200044},{"active_seconds":250.0639041190011,"checkpoint_seconds":244.30086884399998,"interface_bytes":{"docker0":{"rx":0,"tx":0},"enp39s0":{"rx":82788650081,"tx":82940184787}},"loop_seconds":494.5044035430001,"rank":1,"sent_bytes":82940184787,"synchronization_seconds":180.32395741599976,"update_seconds":69.73994670300135},{"active_seconds":250.08133555600398,"checkpoint_seconds":244.30001328399976,"interface_bytes":{"docker0":{"rx":0,"tx":0},"enp39s0":{"rx":82773524630,"tx":82939633515}},"loop_seconds":494.5050013590003,"rank":2,"sent_bytes":82939633515,"synchronization_seconds":180.56740389700053,"update_seconds":69.51393165900345},{"active_seconds":250.06704056900253,"checkpoint_seconds":244.30106188499985,"interface_bytes":{"docker0":{"rx":0,"tx":0},"enp39s0":{"rx":82779410227,"tx":82942983091}},"loop_seconds":494.50094131399965,"rank":3,"sent_bytes":82942983091,"synchronization_seconds":179.54220891900013,"update_seconds":70.5248316500024}],"total_sent_bytes":331777955576,"world":4},"single":{"allocated_gpu_loop_seconds":266.116835575,"max_rank_active_seconds":208.08809893700038,"max_rank_loop_seconds":266.116835575,"parameter_digest":"e6c78787eb997e9f3793dfcc29e4bb85e3551aae0fb6cc54df0f66e31273ff11","ranks":[{"active_seconds":208.08809893700038,"checkpoint_seconds":57.90815794499997,"interface_bytes":{"docker0":{"rx":0,"tx":0},"enp39s0":{"rx":19718,"tx":20476}},"loop_seconds":266.116835575,"rank":0,"sent_bytes":20476,"synchronization_seconds":0,"update_seconds":208.08809893700038}],"total_sent_bytes":20476,"world":1}},"conditions":{"active_time":true,"sent_bytes":true},"efficiency_contract_passed":true,"network_scope":"Whole-host interface counters over each rank training loop, excluding loopback; includes checkpoint acknowledgments and SSH traffic, not isolated NCCL bytes","ratios":{"active_seconds_vs_ddp":0.12635246891416219,"active_seconds_vs_single":1.2018176951230373,"loop_seconds_vs_single":1.858225182662799,"sent_bytes_vs_ddp":0.06223953253964159},"scope":"Matched four-host topology, data and global batch; active time includes local updates and synchronization, with checkpoint and whole-process cost reported separately","timing_scope":"Maximum over ranks of summed local update and sync durations. Checkpoint and loop durations are separate; initial model loading, process startup and idle allocation require the process and EC2 receipts."}}