{ "checks": { "closed_loop_raw_histogram_is_more_optimistic": true, "fanout_turns_rare_leaf_tail_into_request_tail": true, "omission_correction_is_not_an_actual_request_log": true, "one_serial_hop_controls_end_to_end_tail": true, "percentiles_are_not_linearly_aggregatable": true, "queue_span_changes_attribution": true, "wall_clock_order_differs_from_causal_parent_order": true }, "failed_checks": [], "fanout": { "deterministic_runs": { "1": { "completion": "all leaves, so request latency is max(leaf latency)", "max_ms": 50.0, "p50_ms": 2.0, "p95_ms": 2.0, "p99_ms": 2.0, "samples": 10000, "slow_leaf_fraction": 0.001 }, "10": { "completion": "all leaves, so request latency is max(leaf latency)", "max_ms": 50.0, "p50_ms": 2.0, "p95_ms": 2.0, "p99_ms": 2.0, "samples": 10000, "slow_leaf_fraction": 0.001 }, "100": { "completion": "all leaves, so request latency is max(leaf latency)", "max_ms": 50.0, "p50_ms": 2.0, "p95_ms": 50.0, "p99_ms": 50.0, "samples": 10000, "slow_leaf_fraction": 0.001 } }, "independent_reference_probability_at_least_one_slow_leaf_width_100": 0.09520785288629108, "marginal_slow_probability_target": 0.001 }, "load_models": { "closed_loop_expected_interval_correction": { "max_ms": 100.0, "p50_ms": 5.0, "p95_ms": 40.0, "p99_ms": 88.0, "samples": 1075, "synthetic_not_actual_requests": true }, "closed_loop_one_outstanding": { "achieved_requests_per_second": 182.64840182648402, "max_ms": 100.0, "offered_load_is_response_gated": true, "p50_ms": 5.0, "p95_ms": 5.0, "p99_ms": 5.0, "samples": 1000 }, "open_loop": { "achieved_requests_per_second": 164.09583196586806, "max_ms": 100.0, "offered_requests_per_second": 166.66666666666666, "p50_ms": 5.0, "p95_ms": 88.0, "p99_ms": 98.0, "samples": 1000 }, "workload": { "fixed_arrival_interval_ms": 6.0, "requests": 1000, "service_pattern": "5 ms normally; every 200th request takes 100 ms" } }, "method": { "production_claim": false, "quantile": "nearest-rank", "randomness": "none; all schedules are deterministic", "time_unit": "synthetic milliseconds" }, "mutant": "none", "percentile_aggregation": { "arithmetic_mean_of_instance_p99_ms": 50.5, "instance_p99_ms": [ 1.0, 100.0 ], "p99_after_merging_samples_ms": 100.0, "quantile_method": "nearest-rank" }, "python": "3.12.3", "serial_path": { "end_to_end": { "max_ms": 51.0, "p50_ms": 8.0, "p95_ms": 8.0, "p99_ms": 51.0, "samples": 2000 }, "hops": [ { "max_ms": 2.0, "p50_ms": 2.0, "p95_ms": 2.0, "p99_ms": 2.0, "samples": 2000 }, { "max_ms": 2.0, "p50_ms": 2.0, "p95_ms": 2.0, "p99_ms": 2.0, "samples": 2000 }, { "max_ms": 45.0, "p50_ms": 2.0, "p95_ms": 2.0, "p99_ms": 45.0, "samples": 2000 }, { "max_ms": 2.0, "p50_ms": 2.0, "p95_ms": 2.0, "p99_ms": 2.0, "samples": 2000 } ], "model": "four serial hops; hop 2 is 45 ms on every 40th request" }, "status": "PASS", "tracing": { "causal_parent_order": [ "root", "gateway", "worker", "queue_wait", "handler", "database" ], "complete_spans": [ { "clock_offset_ms": 0, "duration_ms": 75, "host": "client", "name": "root", "parent": null, "recorded_end_ms": 75, "recorded_start_ms": 0, "true_end_ms": 75, "true_start_ms": 0 }, { "clock_offset_ms": 20, "duration_ms": 8, "host": "gateway", "name": "gateway", "parent": "root", "recorded_end_ms": 28, "recorded_start_ms": 20, "true_end_ms": 8, "true_start_ms": 0 }, { "clock_offset_ms": -25, "duration_ms": 62, "host": "worker", "name": "worker", "parent": "gateway", "recorded_end_ms": 45, "recorded_start_ms": -17, "true_end_ms": 70, "true_start_ms": 8 }, { "clock_offset_ms": -25, "duration_ms": 40, "host": "worker", "name": "queue_wait", "parent": "worker", "recorded_end_ms": 23, "recorded_start_ms": -17, "true_end_ms": 48, "true_start_ms": 8 }, { "clock_offset_ms": -25, "duration_ms": 22, "host": "worker", "name": "handler", "parent": "worker", "recorded_end_ms": 45, "recorded_start_ms": 23, "true_end_ms": 70, "true_start_ms": 48 }, { "clock_offset_ms": 40, "duration_ms": 10, "host": "database", "name": "database", "parent": "handler", "recorded_end_ms": 105, "recorded_start_ms": 95, "true_end_ms": 65, "true_start_ms": 55 } ], "limits": [ "parent identifiers carry causal structure; cross-host wall clocks do not prove it", "a missing span can mean sampling, propagation, instrumentation, export, or storage loss", "a visible span duration alone does not reveal queue time unless that wait is instrumented" ], "naive_recorded_start_order": [ "worker", "queue_wait", "root", "gateway", "handler", "database" ], "propagation_break": { "observation": "one execution becomes two trace fragments", "records": [ { "span": "root", "trace_id": "trace-a" }, { "span": "gateway", "trace_id": "trace-a" }, { "span": "worker", "trace_id": "trace-b" }, { "span": "handler", "trace_id": "trace-b" } ] }, "queue_attribution": { "handler_ms": 22, "queue_wait_ms_when_instrumented": 40, "spans_without_queue": [ { "clock_offset_ms": 0, "duration_ms": 75, "host": "client", "name": "root", "parent": null, "recorded_end_ms": 75, "recorded_start_ms": 0, "true_end_ms": 75, "true_start_ms": 0 }, { "clock_offset_ms": 20, "duration_ms": 8, "host": "gateway", "name": "gateway", "parent": "root", "recorded_end_ms": 28, "recorded_start_ms": 20, "true_end_ms": 8, "true_start_ms": 0 }, { "clock_offset_ms": -25, "duration_ms": 62, "host": "worker", "name": "worker", "parent": "gateway", "recorded_end_ms": 45, "recorded_start_ms": -17, "true_end_ms": 70, "true_start_ms": 8 }, { "clock_offset_ms": -25, "duration_ms": 22, "host": "worker", "name": "handler", "parent": "worker", "recorded_end_ms": 45, "recorded_start_ms": 23, "true_end_ms": 70, "true_start_ms": 48 }, { "clock_offset_ms": 40, "duration_ms": 10, "host": "database", "name": "database", "parent": "handler", "recorded_end_ms": 105, "recorded_start_ms": 95, "true_end_ms": 65, "true_start_ms": 55 } ], "unattributed_inside_worker_ms_without_queue_span": 40, "worker_total_ms": 62 } } }