{"generated_at":"2026-10-16T22:09:29Z","command":".venv/bin/python benchmarks/run_benchmark.py","machine":{"cpu":"Apple M2","logical_cpus":8,"ram_gib":24,"load_avg_1_5_15_at_start":[7.7,10.4,14.6],"os":"macOS-27.0.1-arm64-arm-64bit-Mach-O","python":"3.13.3","onnxruntime":"1.31.0","fastembed":"0.9.0","zstandard":"0.25.0","trafilatura":"2.3.1"},"cold_start":{"embed_model_load_and_first_call_ms":104,"rerank_model_load_and_first_call_ms":87.1},"zstd":{"level":19,"train_docs":40,"held_out_docs":10,"dictionary_bytes":105071,"held_out_raw_bytes":29187,"held_out_dict_bytes":11055,"held_out_plain_bytes":14152,"ratio_dictionary":2.64,"ratio_plain":2.06,"held_out_stored_bytes":14752,"ratio_stored_base64":1.98},"quantization":{"dim":384,"vectors_projected":250000000,"formula":"raw bytes = vectors x bytes_per_vector; float32 = dim*4, uint8 = dim, 1-bit = ceil(dim/8)","bytes_per_vector":{"float32":1536,"uint8":384,"1-bit":48},"raw_vector_bytes":{"float32":384000000000,"uint8":96000000000,"1-bit":12000000000},"recall":{"held-out doc chunks vs train-doc chunks":{"queries":54,"corpus_vectors":206,"recall_at_10":{"binary only":0.522,"binary top-20 + float rescore":0.717,"binary top-40 + float rescore":0.861}},"natural-language questions vs all chunks":{"queries":29,"corpus_vectors":260,"recall_at_10":{"binary only":0.628,"binary top-20 + float rescore":0.79,"binary top-40 + float rescore":0.903}}}},"pruning":{"pairs":29,"mean_raw_tokens":760.8,"mean_chunks_per_doc":5.66,"tokens_saved_pct_mean":69.5,"tokens_saved_pct_median":79,"tokens_saved_pct_min":13.1,"tokens_saved_pct_max":85.5,"top1_hit_rate":0.862,"top3_hit_rate":0.966,"pruned_context_hit_rate":0.931,"random_chunk_baseline":0.18,"cases":[{"doc":"tutorial-nginx-rate-limiting.md","query":"what happens to a burst of requests when nodelay is set on limit_req","chunks":6,"raw_tokens":708,"pruned_tokens":522,"tokens_saved_pct":26.3,"top1_contains_answer":false,"top3_contains_answer":false,"pruned_context_contains_answer":true,"top1_score":0.977},{"doc":"tutorial-python-asyncio-timeouts.md","query":"how do I put a timeout on a blocking requests.get call inside asyncio","chunks":6,"raw_tokens":871,"pruned_tokens":757,"tokens_saved_pct":13.1,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.999},{"doc":"tutorial-git-bisect.md","query":"how to save a git bisect session and restore it after a mistake","chunks":5,"raw_tokens":666,"pruned_tokens":438,"tokens_saved_pct":34.2,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.999},{"doc":"tutorial-postgres-partial-indexes.md","query":"why does postgres ignore my partial index with a prepared statement","chunks":5,"raw_tokens":788,"pruned_tokens":342,"tokens_saved_pct":56.6,"top1_contains_answer":false,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.937},{"doc":"tutorial-redis-streams-consumer-groups.md","query":"how do I take over messages a crashed consumer never acknowledged","chunks":6,"raw_tokens":823,"pruned_tokens":298,"tokens_saved_pct":63.8,"top1_contains_answer":false,"top3_contains_answer":true,"pruned_context_contains_answer":false,"top1_score":0.969},{"doc":"tutorial-kubernetes-probes.md","query":"how should a pod shut down gracefully on SIGTERM behind a load balancer","chunks":5,"raw_tokens":884,"pruned_tokens":160,"tokens_saved_pct":81.9,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.969},{"doc":"tutorial-docker-multistage-builds.md","query":"how small is a static Go binary image on a distroless base","chunks":5,"raw_tokens":678,"pruned_tokens":106,"tokens_saved_pct":84.4,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.997},{"doc":"tutorial-sql-window-functions.md","query":"find streaks of consecutive active days per user in SQL","chunks":6,"raw_tokens":791,"pruned_tokens":168,"tokens_saved_pct":78.8,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":1},{"doc":"tutorial-typescript-discriminated-unions.md","query":"make the compiler fail when a switch misses a union case","chunks":6,"raw_tokens":828,"pruned_tokens":170,"tokens_saved_pct":79.5,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.969},{"doc":"tutorial-pandas-resampling.md","query":"daily resample around a daylight saving change gives wrong totals","chunks":6,"raw_tokens":803,"pruned_tokens":170,"tokens_saved_pct":78.8,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.934},{"doc":"tutorial-github-actions-matrix.md","query":"generate a GitHub Actions matrix from the output of a previous job","chunks":6,"raw_tokens":891,"pruned_tokens":382,"tokens_saved_pct":57.1,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.996},{"doc":"tutorial-rust-error-handling.md","query":"when is it acceptable to use unwrap or expect in Rust","chunks":6,"raw_tokens":769,"pruned_tokens":155,"tokens_saved_pct":79.8,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.964},{"doc":"api-ref-ledgerline-payments.md","query":"how long is an Idempotency-Key remembered when retrying a POST","chunks":7,"raw_tokens":965,"pruned_tokens":149,"tokens_saved_pct":84.6,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.98},{"doc":"api-ref-ledgerline-payments.md","query":"how many requests per second do live keys allow","chunks":7,"raw_tokens":965,"pruned_tokens":199,"tokens_saved_pct":79.4,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":1},{"doc":"api-ref-keyhaven-oauth.md","query":"what happens if an old refresh token is used again","chunks":6,"raw_tokens":799,"pruned_tokens":116,"tokens_saved_pct":85.5,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.998},{"doc":"api-ref-coldvault-object-storage.md","query":"what is the maximum expiry of a presigned upload URL","chunks":5,"raw_tokens":731,"pruned_tokens":135,"tokens_saved_pct":81.5,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.962},{"doc":"api-ref-tickstream-websocket.md","query":"the server closed my websocket for too many connections, how long should I back off","chunks":6,"raw_tokens":748,"pruned_tokens":146,"tokens_saved_pct":80.5,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.989},{"doc":"api-ref-ringbuf-rust-crate.md","query":"why does Ring::with_capacity(1000) hold more elements than requested","chunks":6,"raw_tokens":752,"pruned_tokens":128,"tokens_saved_pct":83,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.995},{"doc":"faq-coldvault.md","query":"can I restore an object after deleting it","chunks":5,"raw_tokens":677,"pruned_tokens":132,"tokens_saved_pct":80.5,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.969},{"doc":"troubleshooting-fastqueue.md","query":"why did my fastqueue job execute two times","chunks":6,"raw_tokens":729,"pruned_tokens":117,"tokens_saved_pct":84,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.993},{"doc":"guide-ledgerline-v1-to-v2-migration.md","query":"webhook signature verification fails after parsing the JSON body","chunks":6,"raw_tokens":746,"pruned_tokens":157,"tokens_saved_pct":79,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.973},{"doc":"advisory-tempo-fmt-redos.md","query":"how to mitigate the tempo-fmt parse ReDoS without upgrading","chunks":4,"raw_tokens":484,"pruned_tokens":264,"tokens_saved_pct":45.5,"top1_contains_answer":false,"top3_contains_answer":true,"pruned_context_contains_answer":false,"top1_score":0.821},{"doc":"readme-tinykv-c.md","query":"which tinykv write flag syncs data to disk after each write","chunks":5,"raw_tokens":645,"pruned_tokens":117,"tokens_saved_pct":81.9,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.977},{"doc":"readme-schemashift-sqlite.md","query":"how do I change a column type in SQLite when ALTER TABLE cannot","chunks":5,"raw_tokens":678,"pruned_tokens":168,"tokens_saved_pct":75.2,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":1},{"doc":"api-ref-imgpipe-cli.md","query":"what does imgpipe exit code 3 mean","chunks":5,"raw_tokens":666,"pruned_tokens":301,"tokens_saved_pct":54.8,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.966},{"doc":"rfc-stockroom-event-sourcing.md","query":"how does the stock ledger stop stock from going negative on sales","chunks":7,"raw_tokens":846,"pruned_tokens":146,"tokens_saved_pct":82.7,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.104},{"doc":"readme-terraform-vpc-module.md","query":"which NAT gateway mode survives an availability zone outage","chunks":4,"raw_tokens":678,"pruned_tokens":165,"tokens_saved_pct":75.7,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.493},{"doc":"readme-hammerhead-load-tester.md","query":"why do closed-loop load testers hide latency stalls","chunks":6,"raw_tokens":657,"pruned_tokens":207,"tokens_saved_pct":68.5,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.993},{"doc":"explainer-web-streams-api.md","query":"how does backpressure work with pipeThrough in web streams","chunks":6,"raw_tokens":796,"pruned_tokens":333,"tokens_saved_pct":58.2,"top1_contains_answer":true,"top3_contains_answer":true,"pruned_context_contains_answer":true,"top1_score":0.999}]},"latency":{"embed_per_doc":{"n":150,"p50_ms":109.6,"p95_ms":167.49,"unit":"one page (title + first 2,000 Markdown chars) per call"},"rerank_per_20_chunks":{"n":87,"p50_ms":501.45,"p95_ms":573.3,"unit":"one query x 20 random corpus chunks per call"},"prune_per_page":{"n":29,"p50_ms":136.4,"p95_ms":173.86,"unit":"chunk + cross-encode + select one corpus doc"},"html_to_markdown":{"n":100,"p50_ms":4.6,"p95_ms":6.97,"unit":"engine page_to_result (trafilatura) per fixture page","fixtures":[{"fixture":"api-ref-ledgerline-payments.html","html_bytes":9818,"markdown_chars":3746,"n":25,"p50_ms":6.22,"p95_ms":7.18},{"fixture":"blog-postgres-partial-indexes.html","html_bytes":11725,"markdown_chars":3303,"n":25,"p50_ms":3.19,"p95_ms":3.27},{"fixture":"docs-nginx-rate-limiting.html","html_bytes":11597,"markdown_chars":2773,"n":25,"p50_ms":3.28,"p95_ms":3.32},{"fixture":"qa-asyncio-blocking-timeout.html","html_bytes":8804,"markdown_chars":1796,"n":25,"p50_ms":6.76,"p95_ms":7.13}]}},"compression":{"source":"offline fixture","pages":8,"codec_ratio_total":3.12,"codec_ratio_median":2.64,"pipeline_ratio_total":179.74,"pipeline_ratio_median":126.12,"pipeline_ratio_min":9.2,"pipeline_ratio_max":424.2,"label":"codec = Markdown -> zstd (lossless compression); pipeline = raw HTML -> stored zstd Markdown (extraction + compression, lossy: not a compression ratio)","errors":[],"rows":[{"url":"https://fixtures.kilosearch.test/api-ref-ledgerline-payments","html_bytes":9818,"markdown_bytes":3746,"zstd_bytes":150},{"url":"https://fixtures.kilosearch.test/blog-postgres-partial-indexes","html_bytes":11725,"markdown_bytes":3303,"zstd_bytes":1275},{"url":"https://fixtures.kilosearch.test/docs-nginx-rate-limiting","html_bytes":11597,"markdown_bytes":2773,"zstd_bytes":1051},{"url":"https://fixtures.kilosearch.test/qa-asyncio-blocking-timeout","html_bytes":8804,"markdown_bytes":1796,"zstd_bytes":681},{"url":"https://fixtures.kilosearch.test/docs-docusaurus-data-fetching","html_bytes":332150,"markdown_bytes":2161,"zstd_bytes":783},{"url":"https://fixtures.kilosearch.test/docs-sphinx-taskqueue-retries","html_bytes":275123,"markdown_bytes":4197,"zstd_bytes":1473},{"url":"https://fixtures.kilosearch.test/qa-stackexchange-latest-row-per-group","html_bytes":322341,"markdown_bytes":2152,"zstd_bytes":871},{"url":"https://fixtures.kilosearch.test/repo-github-readme-quiver","html_bytes":280683,"markdown_bytes":1584,"zstd_bytes":683}]},"live":{"kilo":"not measured","exa":"not measured","tavily":"not measured"}}