{ "schema_version": 1, "study": { "title": "Qwen3.8-27B local benchmark v2.1", "brand": "Sonny's Labs", "run_date": "2026-10-09", "report_date": "2026-10-10", "model_count": 17, "followup_count": 14, "hardware": "NVIDIA RTX 5070 Ti 16GB", "gpu_vram_mib": 16303, "cpu": "AMD Ryzen 5 5600G", "cpu_threads": 12, "runtime": { "name": "llama.cpp", "version": "0.6.0", "commit": "d81235049384534c167caea52b85a694f6103d14", "image": "firetussin/vastbench-llama@sha256:d9863ee212edb8bc46cdad71e6cdc6747f6931fcb3e9de9dfe1924016118ccf9" }, "context": { "target_tokens": 65536, "allocated_tokens": 73728, "exercised_prompt_tokens": 57135, "recall_validated_tokens": null, "recall_measurement": "14 literal synthetic-dossier markers" }, "practical_completion_cap": 2048, "humaneval_completion_cap": 2048, "core_completion_cap": 3072, "reasoning_budget": 512, "seed": 12345, "practical_repetitions": 3, "practical_responses": 1224, "humaneval_samples": 2788, "started_at": "2026-10-09T14:11:05.670651+00:00", "finished_at": "2026-10-10T04:31:33.774398+00:00", "wall_seconds": 51628, "queue_wall_seconds": 42768, "cost": { "benchmark_cloud_rental_usd": 0, "quant_creation_usd": null, "idle_failed_cloud_rental_usd": 0, "electricity_usd": null, "total_project_cost_usd": null, "scope": "Local hardware; no Vast rental for this study. Electricity, hardware amortization, bandwidth and historic quant creation were not metered. $0 cloud rental is not a zero-cost total project claim." } }, "report_url": "https://report4.firetussin.com", "intended_custom_domain": "https://report4.firetussin.com", "custom_domain_managed_by_user": false, "weighting": { "applied_percent": 90, "humaneval_plus_percent": 10, "category_weights": { "business": 20, "research": 20, "knowledge": 20, "agent_tools": 15, "practical_sysadmin": 5, "humaneval_plus": 10, "executive_content": 10 }, "role_weights": { "CFO / Analyst": { "business": 50, "research": 20, "knowledge": 15, "agent_tools": 10, "executive_content": 5 }, "Researcher / Scout": { "research": 40, "knowledge": 25, "agent_tools": 15, "executive_content": 10, "business": 10 }, "Hermes Coordinator": { "agent_tools": 30, "knowledge": 25, "research": 20, "executive_content": 15, "practical_sysadmin": 5, "humaneval_plus": 5 }, "Librarian / Knowledge": { "knowledge": 35, "research": 25, "agent_tools": 20, "executive_content": 15, "business": 5 }, "Forge / Coder": { "humaneval_plus": 50, "practical_sysadmin": 20, "agent_tools": 15, "knowledge": 10, "research": 5 } }, "formula": "sum(category_score * category_weight) / 100; category means are equally weighted tasks within each category" }, "metric_definitions": { "adaptive_seconds": "Median wall seconds of three adaptive long-context requests; first full-dossier request followed by two requests with prompt-cache reuse. Lower is better; empty finals excluded from speed ranking.", "free_vram_mib": "Minimum free GPU memory sampled across the three adaptive long-context repetitions, in MiB. Higher is more headroom.", "vram_gib": "(16303 MiB physical GPU capacity minus minimum sampled free memory)/1024; GPU-wide observed use, not model file size.", "humaneval_plus_percent": "164 Python candidates per configuration, one sample per problem; isolated EvalPlus 0.3.1 pass@1. Higher is better.", "balanced_score": "Personalized practical/coding index out of 100; speed, VRAM and long-context quality are excluded. Higher is better." }, "models": [ { "id": "qwen38-gsq-rco-iq3s-mtp", "label": "GSQ-RCO · IQ3_S · MTP", "name": "Qwen3.8-27B GSQ-RCO IQ3_S MTP", "provider": "GSQ-RCO", "quant": "IQ3_S", "repo_id": "ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF", "revision": "d562806dbafae37109975e970aae91b43e73b440", "filename": "Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf", "sha256": "58FD826723939933DC86F45B7FE04545CBC2DE1C70F6FE2CDD3858C87A98C12F", "source_url": "https://huggingface.co/ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF/tree/d562806dbafae37109975e970aae91b43e73b440", "file_bytes": 12120016960, "file_gib": 11.288, "mtp": true, "selected_adaptive": { "mtp": 3, "ubatch": 256 }, "practical_profile": { "ctx": 73728, "ubatch": 256, "mtp": 2 }, "balanced_score": 97.7, "category_scores": { "business": 100.0, "research": 100.0, "knowledge": 100.0, "agent_tools": 100.0, "practical_sysadmin": 95.0, "executive_content": 89.285, "humaneval_plus": 90.24 }, "role_scores": { "CFO / Analyst": 99.46, "Researcher / Scout": 98.93, "Hermes Coordinator": 97.65, "Librarian / Knowledge": 98.39, "Forge / Coder": 94.12 }, "applied_weighted_score": 98.53, "humaneval_plus_passed": 148, "humaneval_base_passed": 152, "humaneval_evaluated": 164, "humaneval_plus_percent": 90.24, "humaneval_base_percent": 92.68, "adaptive_seconds": 31.5127, "fixed_seconds": 33.9472, "successful_long_context_output": true, "speed_rank_eligible": true, "long_context_empty_finals": 0, "adaptive_first_request_seconds": 84.3482, "adaptive_repetitions_seconds": [ 84.3482, 31.489, 31.5127 ], "adaptive_generated_tokens": [ 2271, 2271, 2271 ], "adaptive_prompt_tokens": [ 57135, 57135, 57135 ], "adaptive_cached_prompt_tokens": [ 0, 57131, 57131 ], "adaptive_generation_tps_median": 73.77, "adaptive_quality": [ { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 875 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 875 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 875 } ], "fixed_quality": [ { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 861 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 776 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 861 } ], "free_vram_mib": 1628.0, "vram_gib": 14.331, "peak_temperature_c": 63.0, "allocation_max_tokens": 131072, "context_allocated_tokens": 73728, "context_exercised_prompt_tokens": 57135, "recall_marker_hits_adaptive": [ 14, 14, 14 ], "practical_responses": 72, "practical_empty": 0, "practical_length_finishes": 3, "practical_repeat_identical_tasks": 24, "run_id": "20261009-141105-local-5070ti-v2.1-full", "scorer_image_id": "sha256:b06643a7ed28f7cd6532c8c8b88bb5a6990460fddd2b62da845d3c66fba2d45a", "scorer_rebuilt": false, "scorer_recovery": null, "run_completed_at": "2026-10-09T16:38:39.681447+00:00", "rank": 1 }, { "id": "qwen38-hauhaucs-uncensored-q2-k-p", "label": "HauhauCS Aggressive · Q2_K_P · MTP", "name": "Qwen3.8-27B HauhauCS Uncensored Q2_K_P MTP", "provider": "HauhauCS Aggressive", "quant": "Q2_K_P", "repo_id": "HauhauCS/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-MTP-GGUF", "revision": "993a5971fda8f30dd1b7eb2654792ba4415c7460", "filename": "Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q2_K_P.gguf", "sha256": "F778A15FA0393B38E473B954B39A793543FC424838E7AF8C0F6BD27B6CC11CA7", "source_url": "https://huggingface.co/HauhauCS/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-MTP-GGUF/tree/993a5971fda8f30dd1b7eb2654792ba4415c7460", "file_bytes": 10676422784, "file_gib": 9.943, "mtp": true, "selected_adaptive": { "mtp": 3, "ubatch": 512 }, "practical_profile": { "ctx": 73728, "ubatch": 256, "mtp": 2 }, "balanced_score": 97.61, "category_scores": { "business": 100.0, "research": 100.0, "knowledge": 100.0, "agent_tools": 100.0, "practical_sysadmin": 100.0, "executive_content": 90.0775, "humaneval_plus": 85.98 }, "role_scores": { "CFO / Analyst": 99.5, "Researcher / Scout": 99.01, "Hermes Coordinator": 97.81, "Librarian / Knowledge": 98.51, "Forge / Coder": 92.99 }, "applied_weighted_score": 98.9, "humaneval_plus_passed": 141, "humaneval_base_passed": 144, "humaneval_evaluated": 164, "humaneval_plus_percent": 85.98, "humaneval_base_percent": 87.8, "adaptive_seconds": 6.8971, "fixed_seconds": 7.7443, "successful_long_context_output": false, "speed_rank_eligible": false, "long_context_empty_finals": 6, "adaptive_first_request_seconds": 56.98, "adaptive_repetitions_seconds": [ 56.98, 6.8971, 6.8881 ], "adaptive_generated_tokens": [ 515, 515, 515 ], "adaptive_prompt_tokens": [ 57135, 57135, 57135 ], "adaptive_cached_prompt_tokens": [ 0, 57131, 57131 ], "adaptive_generation_tps_median": 83.37, "adaptive_quality": [ { "overall_hermes": 32.9, "retrieval_facts": "0/14", "word_count": 0 }, { "overall_hermes": 32.9, "retrieval_facts": "0/14", "word_count": 0 }, { "overall_hermes": 32.9, "retrieval_facts": "0/14", "word_count": 0 } ], "fixed_quality": [ { "overall_hermes": 32.9, "retrieval_facts": "0/14", "word_count": 0 }, { "overall_hermes": 32.9, "retrieval_facts": "0/14", "word_count": 0 }, { "overall_hermes": 32.9, "retrieval_facts": "0/14", "word_count": 0 } ], "free_vram_mib": 3026.0, "vram_gib": 12.966, "peak_temperature_c": 64.0, "allocation_max_tokens": 131072, "context_allocated_tokens": 73728, "context_exercised_prompt_tokens": 57135, "recall_marker_hits_adaptive": [ 0, 0, 0 ], "practical_responses": 72, "practical_empty": 0, "practical_length_finishes": 0, "practical_repeat_identical_tasks": 24, "run_id": "20261010-021726-v2.1-qwen38-hauhaucs-uncensored-q2-k-p", "scorer_image_id": "sha256:a1ce362575e724db75b4bf9a744fc51a0b16e2c4a267aeca076217b0971272f5", "scorer_rebuilt": true, "scorer_recovery": { "replacement_image_id": "sha256:a1ce362575e724db75b4bf9a744fc51a0b16e2c4a267aeca076217b0971272f5", "dockerfile_sha256": "a2a2f40af7d5046ed496855fb3bb0a07dbd80e7f3a1da355454c0ff211eaa4a9", "reason": "Original pinned scorer image disappeared from host; rebuilt retained identical Dockerfile. Same pinned Python base and EvalPlus 0.3.1; transitive dependencies were not fully locked in original build.", "isolation_probe_passed": true, "recovered_at": "2026-10-10T00:31:47.125849+00:00", "original_image_id": "sha256:b06643a7ed28f7cd6532c8c8b88bb5a6990460fddd2b62da845d3c66fba2d45a" }, "run_completed_at": "2026-10-10T02:59:59.458159+00:00", "rank": 2 }, { "id": "qwen38-pi-iq2m-control", "label": "Pi control · IQ2_M", "name": "Qwen3.8-27B Pi IQ2_M control", "provider": "Pi control", "quant": "IQ2_M", "repo_id": "firetussin/Qwen3.8-27B-pi-RCO-allocation-transfer-GGUF", "revision": "8e4712137774900181d9e7b833eb82a039445bc4", "filename": "Qwen3.8-27B-pi-llama-IQ2_M-base-imatrix.gguf", "sha256": "11DA0AC94BE157B2911A97DAAC130E36692164AABA0C9DAA0D144A9FA7460EC6", "source_url": "https://huggingface.co/firetussin/Qwen3.8-27B-pi-RCO-allocation-transfer-GGUF/tree/8e4712137774900181d9e7b833eb82a039445bc4", "file_bytes": 10004593632, "file_gib": 9.318, "mtp": false, "selected_adaptive": { "mtp": 0, "ubatch": 512 }, "practical_profile": { "ctx": 73728, "ubatch": 1024, "mtp": 0 }, "balanced_score": 97.58, "category_scores": { "business": 100.0, "research": 100.0, "knowledge": 100.0, "agent_tools": 100.0, "practical_sysadmin": 100.0, "executive_content": 92.855, "humaneval_plus": 82.93 }, "role_scores": { "CFO / Analyst": 99.64, "Researcher / Scout": 99.29, "Hermes Coordinator": 98.07, "Librarian / Knowledge": 98.93, "Forge / Coder": 91.47 }, "applied_weighted_score": 99.21, "humaneval_plus_passed": 136, "humaneval_base_passed": 142, "humaneval_evaluated": 164, "humaneval_plus_percent": 82.93, "humaneval_base_percent": 86.59, "adaptive_seconds": 39.5328, "fixed_seconds": 39.5408, "successful_long_context_output": true, "speed_rank_eligible": true, "long_context_empty_finals": 0, "adaptive_first_request_seconds": 93.7959, "adaptive_repetitions_seconds": [ 93.7959, 39.5123, 39.5328 ], "adaptive_generated_tokens": [ 1620, 1620, 1620 ], "adaptive_prompt_tokens": [ 57135, 57135, 57135 ], "adaptive_cached_prompt_tokens": [ 0, 57131, 57131 ], "adaptive_generation_tps_median": 41.55, "adaptive_quality": [ { "overall_hermes": 97.1, "retrieval_facts": "14/14", "word_count": 474 }, { "overall_hermes": 97.1, "retrieval_facts": "14/14", "word_count": 474 }, { "overall_hermes": 97.1, "retrieval_facts": "14/14", "word_count": 474 } ], "fixed_quality": [ { "overall_hermes": 97.1, "retrieval_facts": "14/14", "word_count": 474 }, { "overall_hermes": 97.1, "retrieval_facts": "14/14", "word_count": 474 }, { "overall_hermes": 97.1, "retrieval_facts": "14/14", "word_count": 474 } ], "free_vram_mib": 4726.0, "vram_gib": 11.306, "peak_temperature_c": 64.0, "allocation_max_tokens": 131072, "context_allocated_tokens": 73728, "context_exercised_prompt_tokens": 57135, "recall_marker_hits_adaptive": [ 14, 14, 14 ], "practical_responses": 72, "practical_empty": 0, "practical_length_finishes": 0, "practical_repeat_identical_tasks": 24, "run_id": "20261009-205329-v2.1-qwen38-pi-iq2m-control", "scorer_image_id": "sha256:b06643a7ed28f7cd6532c8c8b88bb5a6990460fddd2b62da845d3c66fba2d45a", "scorer_rebuilt": false, "scorer_recovery": null, "run_completed_at": "2026-10-09T21:51:24.521769+00:00", "rank": 3 }, { "id": "qwen38-pi-rco-transfer-iq3xxs", "label": "Pi transfer · IQ3_XXS", "name": "Qwen3.8-27B Pi RCO allocation-transfer IQ3_XXS", "provider": "Pi transfer", "quant": "IQ3_XXS", "repo_id": "firetussin/Qwen3.8-27B-pi-RCO-allocation-transfer-GGUF", "revision": "8e4712137774900181d9e7b833eb82a039445bc4", "filename": "Qwen3.8-27B-pi-RCO-allocation-transfer-IQ3_XXS.gguf", "sha256": "35134E868FC680F30B9764125B8B07FD359C935DEEFCCB64DB9025FB2936883C", "source_url": "https://huggingface.co/firetussin/Qwen3.8-27B-pi-RCO-allocation-transfer-GGUF/tree/8e4712137774900181d9e7b833eb82a039445bc4", "file_bytes": 10094357472, "file_gib": 9.401, "mtp": false, "selected_adaptive": { "mtp": 0, "ubatch": 512 }, "practical_profile": { "ctx": 73728, "ubatch": 512, "mtp": 0 }, "balanced_score": 97.58, "category_scores": { "business": 100.0, "research": 97.5, "knowledge": 100.0, "agent_tools": 100.0, "practical_sysadmin": 100.0, "executive_content": 93.65, "humaneval_plus": 87.2 }, "role_scores": { "CFO / Analyst": 99.18, "Researcher / Scout": 98.36, "Hermes Coordinator": 97.91, "Librarian / Knowledge": 98.42, "Forge / Coder": 93.47 }, "applied_weighted_score": 98.74, "humaneval_plus_passed": 143, "humaneval_base_passed": 149, "humaneval_evaluated": 164, "humaneval_plus_percent": 87.2, "humaneval_base_percent": 90.85, "adaptive_seconds": 47.0664, "fixed_seconds": 47.0577, "successful_long_context_output": true, "speed_rank_eligible": true, "long_context_empty_finals": 0, "adaptive_first_request_seconds": 97.6601, "adaptive_repetitions_seconds": [ 97.6601, 47.056, 47.0664 ], "adaptive_generated_tokens": [ 1935, 1935, 1935 ], "adaptive_prompt_tokens": [ 57135, 57135, 57135 ], "adaptive_cached_prompt_tokens": [ 0, 57131, 57131 ], "adaptive_generation_tps_median": 41.58, "adaptive_quality": [ { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 708 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 708 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 708 } ], "fixed_quality": [ { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 708 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 708 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 708 } ], "free_vram_mib": 4284.0, "vram_gib": 11.737, "peak_temperature_c": 65.0, "allocation_max_tokens": 131072, "context_allocated_tokens": 73728, "context_exercised_prompt_tokens": 57135, "recall_marker_hits_adaptive": [ 14, 14, 14 ], "practical_responses": 72, "practical_empty": 0, "practical_length_finishes": 0, "practical_repeat_identical_tasks": 24, "run_id": "20261009-141105-local-5070ti-v2.1-full", "scorer_image_id": "sha256:b06643a7ed28f7cd6532c8c8b88bb5a6990460fddd2b62da845d3c66fba2d45a", "scorer_rebuilt": false, "scorer_recovery": null, "run_completed_at": "2026-10-09T16:38:39.681447+00:00", "rank": 3 }, { "id": "qwen38-gsq-rco-iq3xxs-mtp", "label": "GSQ-RCO · IQ3_XXS · MTP", "name": "Qwen3.8-27B GSQ-RCO IQ3_XXS MTP", "provider": "GSQ-RCO", "quant": "IQ3_XXS", "repo_id": "ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF", "revision": "d562806dbafae37109975e970aae91b43e73b440", "filename": "Qwen3.8-27B-GSQ-RCO-IQ3_XXS-mtp.gguf", "sha256": "63F29A2189A6B4CC31F81E093D3856AD293A5583114439F41AE1ED7AF4093262", "source_url": "https://huggingface.co/ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF/tree/d562806dbafae37109975e970aae91b43e73b440", "file_bytes": 10442827840, "file_gib": 9.726, "mtp": true, "selected_adaptive": { "mtp": 2, "ubatch": 256 }, "practical_profile": { "ctx": 73728, "ubatch": 1024, "mtp": 3 }, "balanced_score": 97.37, "category_scores": { "business": 100.0, "research": 100.0, "knowledge": 100.0, "agent_tools": 100.0, "practical_sysadmin": 100.0, "executive_content": 86.5075, "humaneval_plus": 87.2 }, "role_scores": { "CFO / Analyst": 99.33, "Researcher / Scout": 98.65, "Hermes Coordinator": 97.34, "Librarian / Knowledge": 97.98, "Forge / Coder": 93.6 }, "applied_weighted_score": 98.5, "humaneval_plus_passed": 143, "humaneval_base_passed": 150, "humaneval_evaluated": 164, "humaneval_plus_percent": 87.2, "humaneval_base_percent": 91.46, "adaptive_seconds": 30.5218, "fixed_seconds": 33.1036, "successful_long_context_output": true, "speed_rank_eligible": true, "long_context_empty_finals": 0, "adaptive_first_request_seconds": 85.8776, "adaptive_repetitions_seconds": [ 85.8776, 30.5215, 30.5218 ], "adaptive_generated_tokens": [ 2008, 2008, 2008 ], "adaptive_prompt_tokens": [ 57135, 57135, 57135 ], "adaptive_cached_prompt_tokens": [ 0, 57131, 57131 ], "adaptive_generation_tps_median": 67.37, "adaptive_quality": [ { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 696 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 696 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 696 } ], "fixed_quality": [ { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 798 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 798 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 798 } ], "free_vram_mib": 3376.0, "vram_gib": 12.624, "peak_temperature_c": 63.0, "allocation_max_tokens": 131072, "context_allocated_tokens": 73728, "context_exercised_prompt_tokens": 57135, "recall_marker_hits_adaptive": [ 14, 14, 14 ], "practical_responses": 72, "practical_empty": 0, "practical_length_finishes": 0, "practical_repeat_identical_tasks": 24, "run_id": "20261009-234120-v2.1-qwen38-gsq-rco-iq3xxs-mtp", "scorer_image_id": "sha256:a1ce362575e724db75b4bf9a744fc51a0b16e2c4a267aeca076217b0971272f5", "scorer_rebuilt": true, "scorer_recovery": { "replacement_image_id": "sha256:a1ce362575e724db75b4bf9a744fc51a0b16e2c4a267aeca076217b0971272f5", "dockerfile_sha256": "a2a2f40af7d5046ed496855fb3bb0a07dbd80e7f3a1da355454c0ff211eaa4a9", "reason": "Original pinned scorer image disappeared from host; rebuilt retained identical Dockerfile. Same pinned Python base and EvalPlus 0.3.1; transitive dependencies were not fully locked in original build.", "isolation_probe_passed": true, "recovered_at": "2026-10-10T00:31:47.125849+00:00", "original_image_id": "sha256:b06643a7ed28f7cd6532c8c8b88bb5a6990460fddd2b62da845d3c66fba2d45a" }, "run_completed_at": "2026-10-10T00:34:16.518209+00:00", "rank": 5 }, { "id": "qwen38-gsq-rco-iq3xxs", "label": "GSQ-RCO · IQ3_XXS", "name": "Qwen3.8-27B GSQ-RCO IQ3_XXS", "provider": "GSQ-RCO", "quant": "IQ3_XXS", "repo_id": "ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF", "revision": "d562806dbafae37109975e970aae91b43e73b440", "filename": "Qwen3.8-27B-GSQ-RCO-IQ3_XXS.gguf", "sha256": "FDFCB6A29B11188956DFBFD904223588A6C1B77EB250C3E8A36E1BD269DF91F7", "source_url": "https://huggingface.co/ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF/tree/d562806dbafae37109975e970aae91b43e73b440", "file_bytes": 10094357632, "file_gib": 9.401, "mtp": false, "selected_adaptive": { "mtp": 0, "ubatch": 512 }, "practical_profile": { "ctx": 73728, "ubatch": 1024, "mtp": 0 }, "balanced_score": 97.31, "category_scores": { "business": 100.0, "research": 100.0, "knowledge": 100.0, "agent_tools": 100.0, "practical_sysadmin": 100.0, "executive_content": 86.5075, "humaneval_plus": 86.59 }, "role_scores": { "CFO / Analyst": 99.33, "Researcher / Scout": 98.65, "Hermes Coordinator": 97.31, "Librarian / Knowledge": 97.98, "Forge / Coder": 93.3 }, "applied_weighted_score": 98.5, "humaneval_plus_passed": 142, "humaneval_base_passed": 148, "humaneval_evaluated": 164, "humaneval_plus_percent": 86.59, "humaneval_base_percent": 90.24, "adaptive_seconds": 55.2577, "fixed_seconds": 55.3026, "successful_long_context_output": true, "speed_rank_eligible": true, "long_context_empty_finals": 0, "adaptive_first_request_seconds": 105.8465, "adaptive_repetitions_seconds": [ 105.8465, 55.2577, 55.2509 ], "adaptive_generated_tokens": [ 2275, 2275, 2275 ], "adaptive_prompt_tokens": [ 57135, 57135, 57135 ], "adaptive_cached_prompt_tokens": [ 0, 57131, 57131 ], "adaptive_generation_tps_median": 41.57, "adaptive_quality": [ { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 822 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 822 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 822 } ], "fixed_quality": [ { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 822 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 822 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 822 } ], "free_vram_mib": 4284.0, "vram_gib": 11.737, "peak_temperature_c": 65.0, "allocation_max_tokens": 131072, "context_allocated_tokens": 73728, "context_exercised_prompt_tokens": 57135, "recall_marker_hits_adaptive": [ 14, 14, 14 ], "practical_responses": 72, "practical_empty": 0, "practical_length_finishes": 0, "practical_repeat_identical_tasks": 24, "run_id": "20261009-183243-v2.1-qwen38-gsq-rco-iq3xxs", "scorer_image_id": "sha256:b06643a7ed28f7cd6532c8c8b88bb5a6990460fddd2b62da845d3c66fba2d45a", "scorer_rebuilt": false, "scorer_recovery": null, "run_completed_at": "2026-10-09T19:28:02.076048+00:00", "rank": 6 }, { "id": "qwen38-unsloth-ud-q2-k-xl", "label": "Unsloth · UD-Q2_K_XL", "name": "Qwen3.8-27B Unsloth UD-Q2_K_XL", "provider": "Unsloth", "quant": "UD-Q2_K_XL", "repo_id": "unsloth/Qwen3.8-27B-GGUF", "revision": "4ca720788d1e01f1bff70c033e0d0028fd02e502", "filename": "Qwen3.8-27B-UD-Q2_K_XL.gguf", "sha256": "FD4730DD8AAD070517978752B63D530AEB1740D2283CAB9FA24F1E404032DDB0", "source_url": "https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/tree/4ca720788d1e01f1bff70c033e0d0028fd02e502", "file_bytes": 9828981664, "file_gib": 9.154, "mtp": false, "selected_adaptive": { "mtp": 0, "ubatch": 512 }, "practical_profile": { "ctx": 73728, "ubatch": 1024, "mtp": 0 }, "balanced_score": 97.09, "category_scores": { "business": 100.0, "research": 100.0, "knowledge": 100.0, "agent_tools": 97.5, "practical_sysadmin": 100.0, "executive_content": 89.285, "humaneval_plus": 85.37 }, "role_scores": { "CFO / Analyst": 99.21, "Researcher / Scout": 98.55, "Hermes Coordinator": 96.91, "Librarian / Knowledge": 97.89, "Forge / Coder": 92.31 }, "applied_weighted_score": 98.39, "humaneval_plus_passed": 140, "humaneval_base_passed": 144, "humaneval_evaluated": 164, "humaneval_plus_percent": 85.37, "humaneval_base_percent": 87.8, "adaptive_seconds": 42.3193, "fixed_seconds": 42.2704, "successful_long_context_output": true, "speed_rank_eligible": true, "long_context_empty_finals": 0, "adaptive_first_request_seconds": 92.595, "adaptive_repetitions_seconds": [ 92.595, 42.3117, 42.3193 ], "adaptive_generated_tokens": [ 1772, 1772, 1772 ], "adaptive_prompt_tokens": [ 57135, 57135, 57135 ], "adaptive_cached_prompt_tokens": [ 0, 57131, 57131 ], "adaptive_generation_tps_median": 42.41, "adaptive_quality": [ { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 636 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 636 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 636 } ], "fixed_quality": [ { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 636 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 636 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 636 } ], "free_vram_mib": 4954.0, "vram_gib": 11.083, "peak_temperature_c": 65.0, "allocation_max_tokens": 131072, "context_allocated_tokens": 73728, "context_exercised_prompt_tokens": 57135, "recall_marker_hits_adaptive": [ 14, 14, 14 ], "practical_responses": 72, "practical_empty": 0, "practical_length_finishes": 0, "practical_repeat_identical_tasks": 24, "run_id": "20261009-163846-v2.1-qwen38-unsloth-ud-q2-k-xl", "scorer_image_id": "sha256:b06643a7ed28f7cd6532c8c8b88bb5a6990460fddd2b62da845d3c66fba2d45a", "scorer_rebuilt": false, "scorer_recovery": null, "run_completed_at": "2026-10-09T17:32:58.393493+00:00", "rank": 7 }, { "id": "qwen38-orcarouter-uncensored-iq3-xxs", "label": "OrcaRouter · IQ3_XXS · MTP", "name": "Qwen3.8-27B orcarouter Uncensored IQ3_XXS MTP", "provider": "OrcaRouter", "quant": "IQ3_XXS", "repo_id": "orcarouter/Qwen3.8-27B-Uncensored-GGUF", "revision": "fc437a3374c9977bdc339a8ec106dcd9a0357001", "filename": "Qwen3.8-27B-Uncensored-IQ3_XXS.gguf", "sha256": "0D91AD7B2C57F545A030B1D391375A5F9A822423B90923D91CF744E717E80D03", "source_url": "https://huggingface.co/orcarouter/Qwen3.8-27B-Uncensored-GGUF/tree/fc437a3374c9977bdc339a8ec106dcd9a0357001", "file_bytes": 11637692000, "file_gib": 10.838, "mtp": true, "selected_adaptive": { "mtp": 3, "ubatch": 256 }, "practical_profile": { "ctx": 73728, "ubatch": 256, "mtp": 2 }, "balanced_score": 97.0, "category_scores": { "business": 100.0, "research": 100.0, "knowledge": 100.0, "agent_tools": 97.5, "practical_sysadmin": 100.0, "executive_content": 86.5075, "humaneval_plus": 87.2 }, "role_scores": { "CFO / Analyst": 99.08, "Researcher / Scout": 98.28, "Hermes Coordinator": 96.59, "Librarian / Knowledge": 97.48, "Forge / Coder": 93.22 }, "applied_weighted_score": 98.08, "humaneval_plus_passed": 143, "humaneval_base_passed": 149, "humaneval_evaluated": 164, "humaneval_plus_percent": 87.2, "humaneval_base_percent": 90.85, "adaptive_seconds": 26.2816, "fixed_seconds": 32.8604, "successful_long_context_output": true, "speed_rank_eligible": true, "long_context_empty_finals": 0, "adaptive_first_request_seconds": 78.7592, "adaptive_repetitions_seconds": [ 78.7592, 26.2816, 26.2473 ], "adaptive_generated_tokens": [ 1847, 1847, 1847 ], "adaptive_prompt_tokens": [ 57135, 57135, 57135 ], "adaptive_cached_prompt_tokens": [ 0, 57131, 57131 ], "adaptive_generation_tps_median": 72.35, "adaptive_quality": [ { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 692 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 692 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 692 } ], "fixed_quality": [ { "overall_hermes": 97.1, "retrieval_facts": "14/14", "word_count": 775 }, { "overall_hermes": 97.1, "retrieval_facts": "14/14", "word_count": 775 }, { "overall_hermes": 97.1, "retrieval_facts": "14/14", "word_count": 775 } ], "free_vram_mib": 2436.0, "vram_gib": 13.542, "peak_temperature_c": 64.0, "allocation_max_tokens": 131072, "context_allocated_tokens": 73728, "context_exercised_prompt_tokens": 57135, "recall_marker_hits_adaptive": [ 14, 14, 14 ], "practical_responses": 72, "practical_empty": 0, "practical_length_finishes": 0, "practical_repeat_identical_tasks": 24, "run_id": "20261010-034645-v2.1-qwen38-orcarouter-uncensored-iq3-xxs", "scorer_image_id": "sha256:a1ce362575e724db75b4bf9a744fc51a0b16e2c4a267aeca076217b0971272f5", "scorer_rebuilt": true, "scorer_recovery": { "replacement_image_id": "sha256:a1ce362575e724db75b4bf9a744fc51a0b16e2c4a267aeca076217b0971272f5", "dockerfile_sha256": "a2a2f40af7d5046ed496855fb3bb0a07dbd80e7f3a1da355454c0ff211eaa4a9", "reason": "Original pinned scorer image disappeared from host; rebuilt retained identical Dockerfile. Same pinned Python base and EvalPlus 0.3.1; transitive dependencies were not fully locked in original build.", "isolation_probe_passed": true, "recovered_at": "2026-10-10T00:31:47.125849+00:00", "original_image_id": "sha256:b06643a7ed28f7cd6532c8c8b88bb5a6990460fddd2b62da845d3c66fba2d45a" }, "run_completed_at": "2026-10-10T04:31:32.813593+00:00", "rank": 8 }, { "id": "qwen38-hauhaucs-uncensored-iq3-xs", "label": "HauhauCS Aggressive · IQ3_XS · MTP", "name": "Qwen3.8-27B HauhauCS Uncensored IQ3_XS MTP", "provider": "HauhauCS Aggressive", "quant": "IQ3_XS", "repo_id": "HauhauCS/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-MTP-GGUF", "revision": "993a5971fda8f30dd1b7eb2654792ba4415c7460", "filename": "Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-IQ3_XS.gguf", "sha256": "E22208325E05D8561B51829048E52512DA57C2BF8D54866199C4126C45BD3504", "source_url": "https://huggingface.co/HauhauCS/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-MTP-GGUF/tree/993a5971fda8f30dd1b7eb2654792ba4415c7460", "file_bytes": 12175558784, "file_gib": 11.339, "mtp": true, "selected_adaptive": { "mtp": 3, "ubatch": 1024 }, "practical_profile": { "ctx": 73728, "ubatch": 256, "mtp": 2 }, "balanced_score": 96.77, "category_scores": { "business": 95.0, "research": 100.0, "knowledge": 100.0, "agent_tools": 100.0, "practical_sysadmin": 95.0, "executive_content": 93.65, "humaneval_plus": 86.59 }, "role_scores": { "CFO / Analyst": 97.18, "Researcher / Scout": 98.86, "Hermes Coordinator": 98.13, "Librarian / Knowledge": 98.8, "Forge / Coder": 92.3 }, "applied_weighted_score": 97.91, "humaneval_plus_passed": 142, "humaneval_base_passed": 148, "humaneval_evaluated": 164, "humaneval_plus_percent": 86.59, "humaneval_base_percent": 90.24, "adaptive_seconds": 28.0724, "fixed_seconds": 34.379, "successful_long_context_output": true, "speed_rank_eligible": true, "long_context_empty_finals": 0, "adaptive_first_request_seconds": 77.6575, "adaptive_repetitions_seconds": [ 77.6575, 28.062, 28.0724 ], "adaptive_generated_tokens": [ 1899, 1899, 1899 ], "adaptive_prompt_tokens": [ 57135, 57135, 57135 ], "adaptive_cached_prompt_tokens": [ 0, 57131, 57131 ], "adaptive_generation_tps_median": 69.44, "adaptive_quality": [ { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 752 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 752 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 752 } ], "fixed_quality": [ { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 809 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 800 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 817 } ], "free_vram_mib": 1434.0, "vram_gib": 14.521, "peak_temperature_c": 62.0, "allocation_max_tokens": 131072, "context_allocated_tokens": 73728, "context_exercised_prompt_tokens": 57135, "recall_marker_hits_adaptive": [ 14, 14, 14 ], "practical_responses": 72, "practical_empty": 0, "practical_length_finishes": 3, "practical_repeat_identical_tasks": 24, "run_id": "20261010-030000-v2.1-qwen38-hauhaucs-uncensored-iq3-xs", "scorer_image_id": "sha256:a1ce362575e724db75b4bf9a744fc51a0b16e2c4a267aeca076217b0971272f5", "scorer_rebuilt": true, "scorer_recovery": { "replacement_image_id": "sha256:a1ce362575e724db75b4bf9a744fc51a0b16e2c4a267aeca076217b0971272f5", "dockerfile_sha256": "a2a2f40af7d5046ed496855fb3bb0a07dbd80e7f3a1da355454c0ff211eaa4a9", "reason": "Original pinned scorer image disappeared from host; rebuilt retained identical Dockerfile. Same pinned Python base and EvalPlus 0.3.1; transitive dependencies were not fully locked in original build.", "isolation_probe_passed": true, "recovered_at": "2026-10-10T00:31:47.125849+00:00", "original_image_id": "sha256:b06643a7ed28f7cd6532c8c8b88bb5a6990460fddd2b62da845d3c66fba2d45a" }, "run_completed_at": "2026-10-10T03:46:36.611105+00:00", "rank": 9 }, { "id": "qwen38-swift-iq2-s", "label": "Swift · IQ2_S", "name": "Swift-Qwen3.8-27B IQ2_S", "provider": "Swift", "quant": "IQ2_S", "repo_id": "ukisai/Swift-Qwen3.8-27B-GGUF", "revision": "f8b396d5df72de36889bd572f3117c2bf70e4911", "filename": "Swift-Qwen3.8-27B-IQ2_S.gguf", "sha256": "9E475E0F1D336F10FECA5B73A3EF1DAB08151D9F5C33E8A75BF911C761E8EF65", "source_url": "https://huggingface.co/ukisai/Swift-Qwen3.8-27B-GGUF/tree/f8b396d5df72de36889bd572f3117c2bf70e4911", "file_bytes": 9896380512, "file_gib": 9.217, "mtp": false, "selected_adaptive": { "mtp": 0, "ubatch": 512 }, "practical_profile": { "ctx": 73728, "ubatch": 1024, "mtp": 0 }, "balanced_score": 96.52, "category_scores": { "business": 100.0, "research": 100.0, "knowledge": 98.215, "agent_tools": 100.0, "practical_sysadmin": 100.0, "executive_content": 90.0775, "humaneval_plus": 78.66 }, "role_scores": { "CFO / Analyst": 99.24, "Researcher / Scout": 98.56, "Hermes Coordinator": 97.0, "Librarian / Knowledge": 97.89, "Forge / Coder": 89.15 }, "applied_weighted_score": 98.5, "humaneval_plus_passed": 129, "humaneval_base_passed": 135, "humaneval_evaluated": 164, "humaneval_plus_percent": 78.66, "humaneval_base_percent": 82.32, "adaptive_seconds": 35.8948, "fixed_seconds": 35.9104, "successful_long_context_output": true, "speed_rank_eligible": true, "long_context_empty_finals": 0, "adaptive_first_request_seconds": 89.3587, "adaptive_repetitions_seconds": [ 89.3587, 35.8948, 35.8928 ], "adaptive_generated_tokens": [ 1517, 1517, 1517 ], "adaptive_prompt_tokens": [ 57135, 57135, 57135 ], "adaptive_cached_prompt_tokens": [ 0, 57131, 57131 ], "adaptive_generation_tps_median": 42.9, "adaptive_quality": [ { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 513 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 513 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 513 } ], "fixed_quality": [ { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 513 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 513 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 513 } ], "free_vram_mib": 5214.0, "vram_gib": 10.829, "peak_temperature_c": 65.0, "allocation_max_tokens": 131072, "context_allocated_tokens": 73728, "context_exercised_prompt_tokens": 57135, "recall_marker_hits_adaptive": [ 14, 14, 14 ], "practical_responses": 72, "practical_empty": 0, "practical_length_finishes": 0, "practical_repeat_identical_tasks": 24, "run_id": "20261009-141105-local-5070ti-v2.1-full", "scorer_image_id": "sha256:b06643a7ed28f7cd6532c8c8b88bb5a6990460fddd2b62da845d3c66fba2d45a", "scorer_rebuilt": false, "scorer_recovery": null, "run_completed_at": "2026-10-09T16:38:39.681447+00:00", "rank": 10 }, { "id": "qwen38-gsq-rco-iq3s", "label": "GSQ-RCO · IQ3_S", "name": "Qwen3.8-27B GSQ-RCO IQ3_S", "provider": "GSQ-RCO", "quant": "IQ3_S", "repo_id": "ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF", "revision": "d562806dbafae37109975e970aae91b43e73b440", "filename": "Qwen3.8-27B-GSQ-RCO-IQ3_S.gguf", "sha256": "64B53B64C7AA39F20A7E54BD80582FE595B1D745624EE8A72E92508C0326D810", "source_url": "https://huggingface.co/ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF/tree/d562806dbafae37109975e970aae91b43e73b440", "file_bytes": 11771546784, "file_gib": 10.963, "mtp": false, "selected_adaptive": { "mtp": 0, "ubatch": 512 }, "practical_profile": { "ctx": 73728, "ubatch": 1024, "mtp": 0 }, "balanced_score": 96.44, "category_scores": { "business": 100.0, "research": 100.0, "knowledge": 100.0, "agent_tools": 90.625, "practical_sysadmin": 100.0, "executive_content": 90.0775, "humaneval_plus": 88.41 }, "role_scores": { "CFO / Analyst": 98.57, "Researcher / Scout": 97.6, "Hermes Coordinator": 95.12, "Librarian / Knowledge": 96.64, "Forge / Coder": 92.8 }, "applied_weighted_score": 97.33, "humaneval_plus_passed": 145, "humaneval_base_passed": 152, "humaneval_evaluated": 164, "humaneval_plus_percent": 88.41, "humaneval_base_percent": 92.68, "adaptive_seconds": 53.7452, "fixed_seconds": 53.7526, "successful_long_context_output": true, "speed_rank_eligible": true, "long_context_empty_finals": 0, "adaptive_first_request_seconds": 101.9073, "adaptive_repetitions_seconds": [ 101.9073, 53.7415, 53.7452 ], "adaptive_generated_tokens": [ 2081, 2081, 2081 ], "adaptive_prompt_tokens": [ 57135, 57135, 57135 ], "adaptive_cached_prompt_tokens": [ 0, 57131, 57131 ], "adaptive_generation_tps_median": 39.11, "adaptive_quality": [ { "overall_hermes": 97.6, "retrieval_facts": "13/14", "word_count": 788 }, { "overall_hermes": 97.6, "retrieval_facts": "13/14", "word_count": 788 }, { "overall_hermes": 97.6, "retrieval_facts": "13/14", "word_count": 788 } ], "fixed_quality": [ { "overall_hermes": 97.6, "retrieval_facts": "13/14", "word_count": 788 }, { "overall_hermes": 97.6, "retrieval_facts": "13/14", "word_count": 788 }, { "overall_hermes": 97.6, "retrieval_facts": "13/14", "word_count": 788 } ], "free_vram_mib": 2684.0, "vram_gib": 13.3, "peak_temperature_c": 65.0, "allocation_max_tokens": 131072, "context_allocated_tokens": 73728, "context_exercised_prompt_tokens": 57135, "recall_marker_hits_adaptive": [ 13, 13, 13 ], "practical_responses": 72, "practical_empty": 0, "practical_length_finishes": 0, "practical_repeat_identical_tasks": 24, "run_id": "20261009-173259-v2.1-qwen38-gsq-rco-iq3s", "scorer_image_id": "sha256:b06643a7ed28f7cd6532c8c8b88bb5a6990460fddd2b62da845d3c66fba2d45a", "scorer_rebuilt": false, "scorer_recovery": null, "run_completed_at": "2026-10-09T18:32:42.022411+00:00", "rank": 11 }, { "id": "qwen38-gsq-rco-iq2s-mtp", "label": "GSQ-RCO · IQ2_S · MTP", "name": "Qwen3.8-27B GSQ-RCO IQ2_S MTP", "provider": "GSQ-RCO", "quant": "IQ2_S", "repo_id": "ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF", "revision": "d562806dbafae37109975e970aae91b43e73b440", "filename": "Qwen3.8-27B-GSQ-RCO-IQ2_S-mtp.gguf", "sha256": "E6406238A5CC0043775CD1963B6F9E5B8707400276E38D9FDE742304906B1330", "source_url": "https://huggingface.co/ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF/tree/d562806dbafae37109975e970aae91b43e73b440", "file_bytes": 9607981120, "file_gib": 8.948, "mtp": true, "selected_adaptive": { "mtp": 2, "ubatch": 1024 }, "practical_profile": { "ctx": 73728, "ubatch": 1024, "mtp": 2 }, "balanced_score": 96.43, "category_scores": { "business": 95.0, "research": 97.5, "knowledge": 100.0, "agent_tools": 100.0, "practical_sysadmin": 95.0, "executive_content": 96.4275, "humaneval_plus": 85.37 }, "role_scores": { "CFO / Analyst": 96.82, "Researcher / Scout": 98.14, "Hermes Coordinator": 97.98, "Librarian / Knowledge": 98.59, "Forge / Coder": 91.56 }, "applied_weighted_score": 97.66, "humaneval_plus_passed": 140, "humaneval_base_passed": 144, "humaneval_evaluated": 164, "humaneval_plus_percent": 85.37, "humaneval_base_percent": 87.8, "adaptive_seconds": 26.9291, "fixed_seconds": 25.8562, "successful_long_context_output": true, "speed_rank_eligible": true, "long_context_empty_finals": 0, "adaptive_first_request_seconds": 79.8838, "adaptive_repetitions_seconds": [ 79.8838, 26.9158, 26.9291 ], "adaptive_generated_tokens": [ 1766, 1766, 1766 ], "adaptive_prompt_tokens": [ 57135, 57135, 57135 ], "adaptive_cached_prompt_tokens": [ 0, 57131, 57131 ], "adaptive_generation_tps_median": 67.38, "adaptive_quality": [ { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 616 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 616 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 616 } ], "fixed_quality": [ { "overall_hermes": 97.6, "retrieval_facts": "13/14", "word_count": 595 }, { "overall_hermes": 97.6, "retrieval_facts": "13/14", "word_count": 595 }, { "overall_hermes": 97.6, "retrieval_facts": "13/14", "word_count": 595 } ], "free_vram_mib": 3660.0, "vram_gib": 12.347, "peak_temperature_c": 63.0, "allocation_max_tokens": 131072, "context_allocated_tokens": 73728, "context_exercised_prompt_tokens": 57135, "recall_marker_hits_adaptive": [ 14, 14, 14 ], "practical_responses": 72, "practical_empty": 0, "practical_length_finishes": 3, "practical_repeat_identical_tasks": 24, "run_id": "20261009-201032-v2.1-qwen38-gsq-rco-iq2s-mtp", "scorer_image_id": "sha256:b06643a7ed28f7cd6532c8c8b88bb5a6990460fddd2b62da845d3c66fba2d45a", "scorer_rebuilt": false, "scorer_recovery": null, "run_completed_at": "2026-10-09T20:53:28.495933+00:00", "rank": 12 }, { "id": "qwen38-gsq-rco-iq2s", "label": "GSQ-RCO · IQ2_S", "name": "Qwen3.8-27B GSQ-RCO IQ2_S", "provider": "GSQ-RCO", "quant": "IQ2_S", "repo_id": "ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF", "revision": "d562806dbafae37109975e970aae91b43e73b440", "filename": "Qwen3.8-27B-GSQ-RCO-IQ2_S.gguf", "sha256": "16C9802111AA9EF3ACDE465188D6D601F8DB128EE3D828AD983A5CACA4135ECB", "source_url": "https://huggingface.co/ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF/tree/d562806dbafae37109975e970aae91b43e73b440", "file_bytes": 9259510912, "file_gib": 8.624, "mtp": false, "selected_adaptive": { "mtp": 0, "ubatch": 512 }, "practical_profile": { "ctx": 73728, "ubatch": 1024, "mtp": 0 }, "balanced_score": 95.54, "category_scores": { "business": 95.0, "research": 95.0, "knowledge": 100.0, "agent_tools": 100.0, "practical_sysadmin": 100.0, "executive_content": 90.0775, "humaneval_plus": 85.37 }, "role_scores": { "CFO / Analyst": 96.0, "Researcher / Scout": 96.51, "Hermes Coordinator": 96.78, "Librarian / Knowledge": 97.01, "Forge / Coder": 92.44 }, "applied_weighted_score": 96.68, "humaneval_plus_passed": 140, "humaneval_base_passed": 146, "humaneval_evaluated": 164, "humaneval_plus_percent": 85.37, "humaneval_base_percent": 89.02, "adaptive_seconds": 35.96, "fixed_seconds": 35.9615, "successful_long_context_output": true, "speed_rank_eligible": true, "long_context_empty_finals": 0, "adaptive_first_request_seconds": 87.1573, "adaptive_repetitions_seconds": [ 87.1573, 35.96, 35.9469 ], "adaptive_generated_tokens": [ 1523, 1523, 1523 ], "adaptive_prompt_tokens": [ 57135, 57135, 57135 ], "adaptive_cached_prompt_tokens": [ 0, 57131, 57131 ], "adaptive_generation_tps_median": 43.0, "adaptive_quality": [ { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 548 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 548 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 548 } ], "fixed_quality": [ { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 548 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 548 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 548 } ], "free_vram_mib": 5080.0, "vram_gib": 10.96, "peak_temperature_c": 65.0, "allocation_max_tokens": 131072, "context_allocated_tokens": 73728, "context_exercised_prompt_tokens": 57135, "recall_marker_hits_adaptive": [ 14, 14, 14 ], "practical_responses": 72, "practical_empty": 0, "practical_length_finishes": 0, "practical_repeat_identical_tasks": 24, "run_id": "20261009-215125-v2.1-qwen38-gsq-rco-iq2s", "scorer_image_id": "sha256:b06643a7ed28f7cd6532c8c8b88bb5a6990460fddd2b62da845d3c66fba2d45a", "scorer_rebuilt": false, "scorer_recovery": null, "run_completed_at": "2026-10-09T22:46:45.106382+00:00", "rank": 13 }, { "id": "qwen38-hauhaucs-uncensored-iq2-m", "label": "HauhauCS Aggressive · IQ2_M · MTP", "name": "Qwen3.8-27B HauhauCS Uncensored IQ2_M MTP", "provider": "HauhauCS Aggressive", "quant": "IQ2_M", "repo_id": "HauhauCS/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-MTP-GGUF", "revision": "993a5971fda8f30dd1b7eb2654792ba4415c7460", "filename": "Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-IQ2_M.gguf", "sha256": "5E14825174E02733B873CC01358787D570B648DD530C061623764ABA743512CE", "source_url": "https://huggingface.co/HauhauCS/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-MTP-GGUF/tree/993a5971fda8f30dd1b7eb2654792ba4415c7460", "file_bytes": 10319906944, "file_gib": 9.611, "mtp": true, "selected_adaptive": { "mtp": 3, "ubatch": 1024 }, "practical_profile": { "ctx": 73728, "ubatch": 256, "mtp": 2 }, "balanced_score": 95.26, "category_scores": { "business": 91.6675, "research": 100.0, "knowledge": 100.0, "agent_tools": 97.5, "practical_sysadmin": 95.0, "executive_content": 86.5075, "humaneval_plus": 89.02 }, "role_scores": { "CFO / Analyst": 94.91, "Researcher / Scout": 97.44, "Hermes Coordinator": 96.43, "Librarian / Knowledge": 97.06, "Forge / Coder": 93.14 }, "applied_weighted_score": 95.95, "humaneval_plus_passed": 146, "humaneval_base_passed": 149, "humaneval_evaluated": 164, "humaneval_plus_percent": 89.02, "humaneval_base_percent": 90.85, "adaptive_seconds": 25.1881, "fixed_seconds": 30.8051, "successful_long_context_output": true, "speed_rank_eligible": true, "long_context_empty_finals": 0, "adaptive_first_request_seconds": 75.5002, "adaptive_repetitions_seconds": [ 75.5002, 25.1881, 25.1864 ], "adaptive_generated_tokens": [ 1817, 1817, 1817 ], "adaptive_prompt_tokens": [ 57135, 57135, 57135 ], "adaptive_cached_prompt_tokens": [ 0, 57131, 57131 ], "adaptive_generation_tps_median": 74.27, "adaptive_quality": [ { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 675 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 675 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 675 } ], "fixed_quality": [ { "overall_hermes": 97.6, "retrieval_facts": "13/14", "word_count": 784 }, { "overall_hermes": 97.6, "retrieval_facts": "13/14", "word_count": 784 }, { "overall_hermes": 97.6, "retrieval_facts": "13/14", "word_count": 784 } ], "free_vram_mib": 3040.0, "vram_gib": 12.952, "peak_temperature_c": 63.0, "allocation_max_tokens": 131072, "context_allocated_tokens": 73728, "context_exercised_prompt_tokens": 57135, "recall_marker_hits_adaptive": [ 14, 14, 14 ], "practical_responses": 72, "practical_empty": 0, "practical_length_finishes": 3, "practical_repeat_identical_tasks": 24, "run_id": "20261010-013048-v2.1-qwen38-hauhaucs-uncensored-iq2-m", "scorer_image_id": "sha256:a1ce362575e724db75b4bf9a744fc51a0b16e2c4a267aeca076217b0971272f5", "scorer_rebuilt": true, "scorer_recovery": { "replacement_image_id": "sha256:a1ce362575e724db75b4bf9a744fc51a0b16e2c4a267aeca076217b0971272f5", "dockerfile_sha256": "a2a2f40af7d5046ed496855fb3bb0a07dbd80e7f3a1da355454c0ff211eaa4a9", "reason": "Original pinned scorer image disappeared from host; rebuilt retained identical Dockerfile. Same pinned Python base and EvalPlus 0.3.1; transitive dependencies were not fully locked in original build.", "isolation_probe_passed": true, "recovered_at": "2026-10-10T00:31:47.125849+00:00", "original_image_id": "sha256:b06643a7ed28f7cd6532c8c8b88bb5a6990460fddd2b62da845d3c66fba2d45a" }, "run_completed_at": "2026-10-10T02:17:25.567390+00:00", "rank": 14 }, { "id": "qwen38-gsq-rco-iq2xs-mtp", "label": "GSQ-RCO · IQ2_XS · MTP", "name": "Qwen3.8-27B GSQ-RCO IQ2_XS MTP", "provider": "GSQ-RCO", "quant": "IQ2_XS", "repo_id": "ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF", "revision": "d562806dbafae37109975e970aae91b43e73b440", "filename": "Qwen3.8-27B-GSQ-RCO-IQ2_XS-mtp.gguf", "sha256": "F3369F8D36968AB7DD12F5FF11452680AEF9371AA19C9922D22CCD5E58C2D9C8", "source_url": "https://huggingface.co/ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF/tree/d562806dbafae37109975e970aae91b43e73b440", "file_bytes": 8771311680, "file_gib": 8.169, "mtp": true, "selected_adaptive": { "mtp": 3, "ubatch": 1024 }, "practical_profile": { "ctx": 73728, "ubatch": 512, "mtp": 3 }, "balanced_score": 95.0, "category_scores": { "business": 95.0, "research": 95.2275, "knowledge": 100.0, "agent_tools": 90.625, "practical_sysadmin": 100.0, "executive_content": 96.4275, "humaneval_plus": 87.2 }, "role_scores": { "CFO / Analyst": 95.43, "Researcher / Scout": 95.83, "Hermes Coordinator": 95.06, "Librarian / Knowledge": 96.15, "Forge / Coder": 91.96 }, "applied_weighted_score": 95.87, "humaneval_plus_passed": 143, "humaneval_base_passed": 152, "humaneval_evaluated": 164, "humaneval_plus_percent": 87.2, "humaneval_base_percent": 92.68, "adaptive_seconds": 23.0666, "fixed_seconds": 24.9506, "successful_long_context_output": true, "speed_rank_eligible": true, "long_context_empty_finals": 0, "adaptive_first_request_seconds": 75.8903, "adaptive_repetitions_seconds": [ 75.8903, 23.0666, 23.0658 ], "adaptive_generated_tokens": [ 1625, 1625, 1625 ], "adaptive_prompt_tokens": [ 57135, 57135, 57135 ], "adaptive_cached_prompt_tokens": [ 0, 57131, 57131 ], "adaptive_generation_tps_median": 72.72, "adaptive_quality": [ { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 589 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 589 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 589 } ], "fixed_quality": [ { "overall_hermes": 95.7, "retrieval_facts": "14/14", "word_count": 592 }, { "overall_hermes": 95.7, "retrieval_facts": "14/14", "word_count": 592 }, { "overall_hermes": 95.7, "retrieval_facts": "14/14", "word_count": 592 } ], "free_vram_mib": 4186.0, "vram_gib": 11.833, "peak_temperature_c": 63.0, "allocation_max_tokens": 131072, "context_allocated_tokens": 73728, "context_exercised_prompt_tokens": 57135, "recall_marker_hits_adaptive": [ 14, 14, 14 ], "practical_responses": 72, "practical_empty": 0, "practical_length_finishes": 0, "practical_repeat_identical_tasks": 24, "run_id": "20261009-192803-v2.1-qwen38-gsq-rco-iq2xs-mtp", "scorer_image_id": "sha256:b06643a7ed28f7cd6532c8c8b88bb5a6990460fddd2b62da845d3c66fba2d45a", "scorer_rebuilt": false, "scorer_recovery": null, "run_completed_at": "2026-10-09T20:10:31.256321+00:00", "rank": 15 }, { "id": "qwen38-unsloth-ud-iq2-s", "label": "Unsloth · UD-IQ2_S", "name": "Qwen3.8-27B Unsloth UD-IQ2_S", "provider": "Unsloth", "quant": "UD-IQ2_S", "repo_id": "unsloth/Qwen3.8-27B-GGUF", "revision": "4ca720788d1e01f1bff70c033e0d0028fd02e502", "filename": "Qwen3.8-27B-UD-IQ2_S.gguf", "sha256": "7897D2C5A5CEE46AEF50895141B2C8A0803C1185F3D03C4FDA4CD137A7AD77FE", "source_url": "https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/tree/4ca720788d1e01f1bff70c033e0d0028fd02e502", "file_bytes": 8371970048, "file_gib": 7.797, "mtp": false, "selected_adaptive": { "mtp": 0, "ubatch": 512 }, "practical_profile": { "ctx": 73728, "ubatch": 1024, "mtp": 0 }, "balanced_score": 94.92, "category_scores": { "business": 100.0, "research": 100.0, "knowledge": 95.0, "agent_tools": 100.0, "practical_sysadmin": 97.5, "executive_content": 87.3, "humaneval_plus": 73.17 }, "role_scores": { "CFO / Analyst": 98.61, "Researcher / Scout": 97.48, "Hermes Coordinator": 95.38, "Librarian / Knowledge": 96.34, "Forge / Coder": 85.58 }, "applied_weighted_score": 97.34, "humaneval_plus_passed": 120, "humaneval_base_passed": 128, "humaneval_evaluated": 164, "humaneval_plus_percent": 73.17, "humaneval_base_percent": 78.05, "adaptive_seconds": 48.487, "fixed_seconds": 48.4827, "successful_long_context_output": true, "speed_rank_eligible": true, "long_context_empty_finals": 0, "adaptive_first_request_seconds": 98.7852, "adaptive_repetitions_seconds": [ 98.7852, 48.4782, 48.487 ], "adaptive_generated_tokens": [ 2120, 2120, 2120 ], "adaptive_prompt_tokens": [ 57135, 57135, 57135 ], "adaptive_cached_prompt_tokens": [ 0, 57131, 57131 ], "adaptive_generation_tps_median": 44.21, "adaptive_quality": [ { "overall_hermes": 97.1, "retrieval_facts": "14/14", "word_count": 716 }, { "overall_hermes": 97.1, "retrieval_facts": "14/14", "word_count": 716 }, { "overall_hermes": 97.1, "retrieval_facts": "14/14", "word_count": 716 } ], "fixed_quality": [ { "overall_hermes": 97.1, "retrieval_facts": "14/14", "word_count": 716 }, { "overall_hermes": 97.1, "retrieval_facts": "14/14", "word_count": 716 }, { "overall_hermes": 97.1, "retrieval_facts": "14/14", "word_count": 716 } ], "free_vram_mib": 5936.0, "vram_gib": 10.124, "peak_temperature_c": 66.0, "allocation_max_tokens": 131072, "context_allocated_tokens": 73728, "context_exercised_prompt_tokens": 57135, "recall_marker_hits_adaptive": [ 14, 14, 14 ], "practical_responses": 72, "practical_empty": 0, "practical_length_finishes": 3, "practical_repeat_identical_tasks": 24, "run_id": "20261010-003417-v2.1-qwen38-unsloth-ud-iq2-s", "scorer_image_id": "sha256:a1ce362575e724db75b4bf9a744fc51a0b16e2c4a267aeca076217b0971272f5", "scorer_rebuilt": true, "scorer_recovery": { "replacement_image_id": "sha256:a1ce362575e724db75b4bf9a744fc51a0b16e2c4a267aeca076217b0971272f5", "dockerfile_sha256": "a2a2f40af7d5046ed496855fb3bb0a07dbd80e7f3a1da355454c0ff211eaa4a9", "reason": "Original pinned scorer image disappeared from host; rebuilt retained identical Dockerfile. Same pinned Python base and EvalPlus 0.3.1; transitive dependencies were not fully locked in original build.", "isolation_probe_passed": true, "recovered_at": "2026-10-10T00:31:47.125849+00:00", "original_image_id": "sha256:b06643a7ed28f7cd6532c8c8b88bb5a6990460fddd2b62da845d3c66fba2d45a" }, "run_completed_at": "2026-10-10T01:30:46.957023+00:00", "rank": 16 }, { "id": "qwen38-gsq-rco-iq2xs", "label": "GSQ-RCO · IQ2_XS", "name": "Qwen3.8-27B GSQ-RCO IQ2_XS", "provider": "GSQ-RCO", "quant": "IQ2_XS", "repo_id": "ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF", "revision": "d562806dbafae37109975e970aae91b43e73b440", "filename": "Qwen3.8-27B-GSQ-RCO-IQ2_XS.gguf", "sha256": "F0AE5006DA0CE6225935339E4E989369F94DE95D2263CF969519F8420C9AE02C", "source_url": "https://huggingface.co/ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF/tree/d562806dbafae37109975e970aae91b43e73b440", "file_bytes": 8422841472, "file_gib": 7.844, "mtp": false, "selected_adaptive": { "mtp": 0, "ubatch": 512 }, "practical_profile": { "ctx": 73728, "ubatch": 1024, "mtp": 0 }, "balanced_score": 93.46, "category_scores": { "business": 86.6675, "research": 97.7275, "knowledge": 100.0, "agent_tools": 90.625, "practical_sysadmin": 95.0, "executive_content": 96.4275, "humaneval_plus": 85.98 }, "role_scores": { "CFO / Analyst": 91.76, "Researcher / Scout": 95.99, "Hermes Coordinator": 95.25, "Librarian / Knowledge": 96.35, "Forge / Coder": 90.47 }, "applied_weighted_score": 94.3, "humaneval_plus_passed": 141, "humaneval_base_passed": 148, "humaneval_evaluated": 164, "humaneval_plus_percent": 85.98, "humaneval_base_percent": 90.24, "adaptive_seconds": 43.3538, "fixed_seconds": 43.3998, "successful_long_context_output": true, "speed_rank_eligible": true, "long_context_empty_finals": 0, "adaptive_first_request_seconds": 95.1739, "adaptive_repetitions_seconds": [ 95.1739, 43.3538, 43.3349 ], "adaptive_generated_tokens": [ 1894, 1894, 1894 ], "adaptive_prompt_tokens": [ 57135, 57135, 57135 ], "adaptive_cached_prompt_tokens": [ 0, 57131, 57131 ], "adaptive_generation_tps_median": 44.25, "adaptive_quality": [ { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 705 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 705 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 705 } ], "fixed_quality": [ { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 705 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 705 }, { "overall_hermes": 98.6, "retrieval_facts": "14/14", "word_count": 705 } ], "free_vram_mib": 5754.0, "vram_gib": 10.302, "peak_temperature_c": 65.0, "allocation_max_tokens": 131072, "context_allocated_tokens": 73728, "context_exercised_prompt_tokens": 57135, "recall_marker_hits_adaptive": [ 14, 14, 14 ], "practical_responses": 72, "practical_empty": 0, "practical_length_finishes": 3, "practical_repeat_identical_tasks": 24, "run_id": "20261009-224646-v2.1-qwen38-gsq-rco-iq2xs", "scorer_image_id": "sha256:b06643a7ed28f7cd6532c8c8b88bb5a6990460fddd2b62da845d3c66fba2d45a", "scorer_rebuilt": false, "scorer_recovery": null, "run_completed_at": "2026-10-09T23:41:19.407719+00:00", "rank": 17 } ], "rankings": { "balanced": [ "qwen38-gsq-rco-iq3s-mtp", "qwen38-hauhaucs-uncensored-q2-k-p", "qwen38-pi-iq2m-control", "qwen38-pi-rco-transfer-iq3xxs", "qwen38-gsq-rco-iq3xxs-mtp", "qwen38-gsq-rco-iq3xxs", "qwen38-unsloth-ud-q2-k-xl", "qwen38-orcarouter-uncensored-iq3-xxs", "qwen38-hauhaucs-uncensored-iq3-xs", "qwen38-swift-iq2-s", "qwen38-gsq-rco-iq3s", "qwen38-gsq-rco-iq2s-mtp", "qwen38-gsq-rco-iq2s", "qwen38-hauhaucs-uncensored-iq2-m", "qwen38-gsq-rco-iq2xs-mtp", "qwen38-unsloth-ud-iq2-s", "qwen38-gsq-rco-iq2xs" ], "successful_speed": [ "qwen38-gsq-rco-iq2xs-mtp", "qwen38-hauhaucs-uncensored-iq2-m", "qwen38-orcarouter-uncensored-iq3-xxs", "qwen38-gsq-rco-iq2s-mtp", "qwen38-hauhaucs-uncensored-iq3-xs", "qwen38-gsq-rco-iq3xxs-mtp", "qwen38-gsq-rco-iq3s-mtp", "qwen38-swift-iq2-s", "qwen38-gsq-rco-iq2s", "qwen38-pi-iq2m-control", "qwen38-unsloth-ud-q2-k-xl", "qwen38-gsq-rco-iq2xs", "qwen38-pi-rco-transfer-iq3xxs", "qwen38-unsloth-ud-iq2-s", "qwen38-gsq-rco-iq3s", "qwen38-gsq-rco-iq3xxs" ], "coding": [ "qwen38-gsq-rco-iq3s-mtp", "qwen38-hauhaucs-uncensored-iq2-m", "qwen38-gsq-rco-iq3s", "qwen38-gsq-rco-iq2xs-mtp", "qwen38-gsq-rco-iq3xxs-mtp", "qwen38-orcarouter-uncensored-iq3-xxs", "qwen38-pi-rco-transfer-iq3xxs", "qwen38-gsq-rco-iq3xxs", "qwen38-hauhaucs-uncensored-iq3-xs", "qwen38-gsq-rco-iq2xs", "qwen38-hauhaucs-uncensored-q2-k-p", "qwen38-gsq-rco-iq2s", "qwen38-gsq-rco-iq2s-mtp", "qwen38-unsloth-ud-q2-k-xl", "qwen38-pi-iq2m-control", "qwen38-swift-iq2-s", "qwen38-unsloth-ud-iq2-s" ] }, "role_recommendations": { "C-Suite / Analyst": { "source_role": "CFO / Analyst", "score": 99.64, "model_ids": [ "qwen38-pi-iq2m-control" ] }, "Researcher / Scout": { "source_role": "Researcher / Scout", "score": 99.29, "model_ids": [ "qwen38-pi-iq2m-control" ] }, "Hermes Coordinator": { "source_role": "Hermes Coordinator", "score": 98.13, "model_ids": [ "qwen38-hauhaucs-uncensored-iq3-xs" ] }, "Librarian / Knowledge": { "source_role": "Librarian / Knowledge", "score": 98.93, "model_ids": [ "qwen38-pi-iq2m-control" ] }, "Forge / Coder": { "source_role": "Forge / Coder", "score": 94.12, "model_ids": [ "qwen38-gsq-rco-iq3s-mtp" ] } }, "mtp_pairs": [ { "quant": "IQ2_XS", "base_id": "qwen38-gsq-rco-iq2xs", "mtp_id": "qwen38-gsq-rco-iq2xs-mtp", "base_seconds": 43.3538, "mtp_seconds": 23.0666, "speedup": 1.879505, "balanced_delta": 1.54, "extra_vram_mib": 1568.0 }, { "quant": "IQ2_S", "base_id": "qwen38-gsq-rco-iq2s", "mtp_id": "qwen38-gsq-rco-iq2s-mtp", "base_seconds": 35.96, "mtp_seconds": 26.9291, "speedup": 1.335358, "balanced_delta": 0.89, "extra_vram_mib": 1420.0 }, { "quant": "IQ3_XXS", "base_id": "qwen38-gsq-rco-iq3xxs", "mtp_id": "qwen38-gsq-rco-iq3xxs-mtp", "base_seconds": 55.2577, "mtp_seconds": 30.5218, "speedup": 1.810434, "balanced_delta": 0.06, "extra_vram_mib": 908.0 }, { "quant": "IQ3_S", "base_id": "qwen38-gsq-rco-iq3s", "mtp_id": "qwen38-gsq-rco-iq3s-mtp", "base_seconds": 53.7452, "mtp_seconds": 31.5127, "speedup": 1.705509, "balanced_delta": 1.26, "extra_vram_mib": 1056.0 } ], "recommendation": { "first_choice": "qwen38-gsq-rco-iq3s-mtp", "reason": "Highest measured balanced score and HumanEval+ pass count, with a successful dossier output; only about 1.59 GiB minimum free VRAM in this workload.", "speed_choice": "qwen38-gsq-rco-iq2xs-mtp", "more_headroom_choice": "qwen38-pi-rco-transfer-iq3xxs" }, "caveats": [ "Balanced index is 90% practical rubric scores and 10% HumanEval+. Long-context quality and speed are separate and do not enter that index.", "HauhauCS Q2_K_P returned empty final answers in all six long-context fixed/adaptive runs, despite API stop finishes. Its 6.90s adaptive median is excluded from successful-speed rankings; its saved 97.61 balanced score is preserved.", "Three fixed-seed repetitions test repeatability; they are not independent trials or confidence intervals. Small score gaps and ties at two-decimal precision are provisional.", "Timing medians include substantial prompt-cache reuse in repetitions 2 and 3. The first full-dossier request is reported separately. Different output lengths also affect wall time.", "Allocation probes use a short prompt. A successful 131072-token allocation is not a 131K recall test. The exercised dossier prompt was 57135 tokens at a 73728-token allocation.", "Retrieval hits use 14 literal markers within a synthetic dossier. They do not establish general long-context recall or overall intelligence.", "Practical scoring revision 2.1 uses deterministic content/structure checks. It has ceiling effects and incomplete semantic coverage; no general-purpose judge model was used.", "The isolated EvalPlus 0.3.1 scorer image was rebuilt after the original image disappeared. Six late configurations use the replacement; Python base and EvalPlus version match, but transitive dependencies were not fully locked. Per-model image IDs and recovery provenance are included.", "Practical profiles are frozen from the anchor audit or historical selections; the four new embedded-MTP models use ctx73728/ubatch256/MTP2. Core tuning remains adaptive. Settings and quant formats differ by configuration.", "HauhauCS uses standard embedded MTP under pinned llama.cpp; the custom FastMTP sidecar and patch were not tested. Uncensored labels are repository labels; this suite did not measure refusal or safety behavior.", "This study used one local RTX 5070 Ti 16GB. Historical report3 used RTX 5060 Ti and older protocol/runtime; do not combine scores or infer runtime-only causes from those differences.", "All model names and repository descriptions are retained as published identifiers. The report independently evaluates the downloaded pinned artifacts, not vendor speed claims." ], "sources": [ { "run_id": "20261009-141105-local-5070ti-v2.1-full", "archive_sha256": "61c0ffd8ddc07ba58e618842fbd81b0b2382761cc4f870b88c60499e29a781bd", "artifacts": { "decision-scores.json": "b6bd66c39e033849e08d8a75f85d2b6b7198b3cbb13d14bae6e2c424dd8efafd", "humaneval-scores.json": "9e2c1090821268522e6666b746d7fcc07c500237f5f48cf0ad4df2b1299a6a11", "summary.json": "002b951409c8a6978185bf76db6e45904e7226176d10cba8c2eb33fa89244efe", "applied-summary.json": "9e57994ef189787ca5b2580798bd717d656c54dc5f65c183ab4997b3c3da99ee", "local-scoring-manifest.json": "6f6f73ee60c37fd3cf76fa70d389a1145a8275f317132ca956547a87bd5ab5a3" } }, { "run_id": "20261009-163846-v2.1-qwen38-unsloth-ud-q2-k-xl", "archive_sha256": "1dbb08271801f6a7c3e7b62a86047b3812c1ef37d0a894348ecfeb4adbf1fd43", "artifacts": { "decision-scores.json": "e342b3c807dd4ca9a090046746634483ed0293e36800f08c4bb235d4d14d4ad8", "humaneval-scores.json": "f346441c63bf059ff352446d6452f7cd0bf69dda7e53ceee87857fde5ca08814", "summary.json": "8d38478989104f276217eba5f0489606838c509f24ac8fa1f923272705a0a4fe", "applied-summary.json": "15537afd0a4cae32e7e94f0e5dd09bece1cce76c666724ea27d65e81152ae5c8", "local-scoring-manifest.json": "2d93e735d2e97915baf0b20f5a7fa712e19b7470511eea2b7bc61f2597c9d889" } }, { "run_id": "20261009-173259-v2.1-qwen38-gsq-rco-iq3s", "archive_sha256": "cc3ce78605fe364007c648f385e1fbd26b372898a8ded269cc581366be550b2e", "artifacts": { "decision-scores.json": "75817a8d835c86d4f948573e24d049ab7eb282b3968bcefbf56e779306b3aafb", "humaneval-scores.json": "06b3f328c5167b10a85c04c9739cea39f4710ba9240e23bc20d1c4cffb00e834", "summary.json": "c18120addab60665b10942ade591470a9224b5458b534adf5aaf6ce3291a9df8", "applied-summary.json": "7b5ae75f68c76c86292eb11d03c7df2c161bc505eeb7ec5ce29ce17bb8407298", "local-scoring-manifest.json": "8a5d70a23bded17ba3d14c17a9d1e8388c1b87ed78f5091545944bd8ef82fa1c" } }, { "run_id": "20261009-183243-v2.1-qwen38-gsq-rco-iq3xxs", "archive_sha256": "15071d9e9d7aa29ead6c2a96a2f2ce0aac5367e75a4cffd9f6d23c91bb9c0f96", "artifacts": { "decision-scores.json": "3e02273db628cc22cacb368c01a43c3d0517b96f3b78c3f731bb7c1f2a923afa", "humaneval-scores.json": "7185a3d8319df412eb4e973d72bf3d0396107e5e40ad1be678e8f79b0ef0ccdd", "summary.json": "35f4a4e580977dc924a06d6c51fee2f63c3ef7c1e3f291e6fd75bb8b00267a16", "applied-summary.json": "7731f10731945ab2ab9e1d1eb47434d353b808ce92cbc82d6bd4e2aeb5127b64", "local-scoring-manifest.json": "2fc605697d2827022dd59ca5d2440ce55f071e178929ebded8055d68d1b02e99" } }, { "run_id": "20261009-192803-v2.1-qwen38-gsq-rco-iq2xs-mtp", "archive_sha256": "e0213e3f2b6bb7cdc0873c340672c43abf4f90426c9facd9917ba1e093a730a6", "artifacts": { "decision-scores.json": "26ec549ccca8989d984a41edf7085f5aa17190e0a274797b6c77ef34b7ab2dd8", "humaneval-scores.json": "da6f92a96e64c3973dc2c85a4449c5b3344a525d919ca57fb7a7fe5ab0c42dcb", "summary.json": "3ff2c58245f8c9eeec107770b3fbaecd4e8310e0c42c72088a6b1d30f9115f4f", "applied-summary.json": "76fd06f6419b05cbceab35289e44e3951ddc0266524c045a704d443f0de470e2", "local-scoring-manifest.json": "bd74487f64f82f89c54564fb26fade24d8ba6fb290aae9d1b303370227f3b083" } }, { "run_id": "20261009-201032-v2.1-qwen38-gsq-rco-iq2s-mtp", "archive_sha256": "6947edc6540fc3ee1dc8c8150892124b2e349c2760c1ff01c485f06a5e2e69af", "artifacts": { "decision-scores.json": "8c246f83e4ba81c05abe3f50dc3f1f1a7d497a25c0427a81b42e38ec9196a5dd", "humaneval-scores.json": "385db41c6d8c61577f0580a09515ed5350c2bb43545ae16c154ee99e889cfec5", "summary.json": "289d6fd365b5a6a37bdf297f851da92f50640eb89887670269d0608db8e95ec4", "applied-summary.json": "37d8328646d76d23de1af8eb55847404508da23b84bcc649ddf625e565cda8cd", "local-scoring-manifest.json": "8700e36c97899bdcc4ecc680f560bd63db1e93819afefc48e16096a74245e628" } }, { "run_id": "20261009-205329-v2.1-qwen38-pi-iq2m-control", "archive_sha256": "ed78b4fecc185cbc4791a798ec4e36e5b8c559643843bcabcaa399b172084425", "artifacts": { "decision-scores.json": "c45c2181cb9358b6feaecd2771822baa8eb236270e6a324273047270f3d26986", "humaneval-scores.json": "19191f2d49f4cab6cce96d9bc29c0a3a868a5eff9f28069eeed6290bfefbb37a", "summary.json": "66fc3d27e0d947a23d1c3a9a82daf4a66f24c94bee57ee363515af122f31b0aa", "applied-summary.json": "8f90691ebcb3ba99c87bce60962351dc00bb003f61667078d04710b70a2050b5", "local-scoring-manifest.json": "548f103ab69ae8b23fea2191b0761c4315312aac2c72c966a2d19c4611432d4e" } }, { "run_id": "20261009-215125-v2.1-qwen38-gsq-rco-iq2s", "archive_sha256": "f0766554107ea9bf5d27029c0788ff54e3364fbf91bdde3591eae2b72002b08e", "artifacts": { "decision-scores.json": "83850a7456825e4cfe96f4b07eadfedf5907d846aa02c6570181045aa772426f", "humaneval-scores.json": "09e9ce68eddde826598072b62f4c76f0b099cb4d235bbb9fbe0ad3b2f713db07", "summary.json": "efd178231cb1866313ee7046209c121d024b0cd7c4ae396559e30ceec890d50f", "applied-summary.json": "426d9ab3cb7dbe2d588e1c3728283814b94180250af2e39f652e75c5f99e1281", "local-scoring-manifest.json": "6fa899323f9ab4ad16a4acb509fc38e39c8142b0118e767e431cf43626390000" } }, { "run_id": "20261009-224646-v2.1-qwen38-gsq-rco-iq2xs", "archive_sha256": "f1b73f10c88dd8190cb9972f04d1ea4065484bac610a0b14f19bc4c96ebddf76", "artifacts": { "decision-scores.json": "d06d430279ec16862dc28a8ce6d8575d2096f01aa9cc92a206a687150ca5b566", "humaneval-scores.json": "f59356e38c1d680b538bd6ee30d515e8ed04d459ef711a4558f76a4a367106da", "summary.json": "39ee71db83d159f1b683b4d809f3903417cc35af0261209e9265127f828374ef", "applied-summary.json": "4d27f9b2e606c82b2b2c29f0e2846b6177d717013566eefbe272a124b6ef1d88", "local-scoring-manifest.json": "1de401fadad93e1fc9828d4b478e3f2c0203061c7e2193b5681cfd24e7940905" } }, { "run_id": "20261009-234120-v2.1-qwen38-gsq-rco-iq3xxs-mtp", "archive_sha256": "0a625c662d770eafba590e8ef0db3896489226eeda4c9d0c8b0dd4e1fb92fe5a", "artifacts": { "decision-scores.json": "b21ed28728d75337915a0b523fd6390d93a505637ee9c8ca6c4b366e07a73c55", "humaneval-scores.json": "084ea52c284d2a5e005266369f4cd325cb9f10a4d4026950892a709f1d62c8f1", "summary.json": "8d4bb3e7475dbd19944b4490033094af51218a554cccbf5395d225274ad28953", "applied-summary.json": "836a30f4ab47b57b2ab9fdc19e3561d9713c6277f4d535f2f783abf0932dce2e", "local-scoring-manifest.json": "0d2807f43b67fa8b1b625fe23a673e5a61616d852b2ec349d3d78d877edffa9d" } }, { "run_id": "20261010-003417-v2.1-qwen38-unsloth-ud-iq2-s", "archive_sha256": "b605e6a4b7416aafe87ce37054f8c1764a0c76ae7bc64b6f8d71ea741e5e1569", "artifacts": { "decision-scores.json": "9e70d3dd7801b5b456876e2348a66a662061a682b4bfe19ba6a289b235d7dd33", "humaneval-scores.json": "7319f01dfb416ae17613f5c6406eb57c6301589608e8ad25ffa9e08cc227f240", "summary.json": "0b43633430575220b7f92968b47a475072ec63f7c79ca0ed3d3e162bbbb2e3ae", "applied-summary.json": "f4a02a9ac203da2e1cfc864ec097244fcae1701c8cbd69e9508955059c114821", "local-scoring-manifest.json": "07ae8639824f99de7a2d18448ce10dcc68d548109290f20615354b6b46b06485" } }, { "run_id": "20261010-034645-v2.1-qwen38-orcarouter-uncensored-iq3-xxs", "archive_sha256": "dfbe1df136809e48bc30d2754ea1c5099ca4754c404a2b1af9458aa2c0d06f63", "artifacts": { "decision-scores.json": "6a24e49cbe70e25c1ec70c570825c2bdfb9334f7d0d107c9f352bb2ba841ebfa", "humaneval-scores.json": "09ff0946f11666eecf3fe514589285aaadba452ce48e636650708d22d112b425", "summary.json": "4231ef9b46754438b934df0f3dab41cd3ed3b01dfa4247d93d21565edde1945a", "applied-summary.json": "099b33b3063d901a550feefeaebc624fbaffe3655f93a57a34c15f13e814d7be", "local-scoring-manifest.json": "65b58746d44696b70ecf9cf928f02554b44e03e6e36e60a5a39a832cf9e7b63a" } }, { "run_id": "20261010-013048-v2.1-qwen38-hauhaucs-uncensored-iq2-m", "archive_sha256": "ca3c62be1902d4fce6e3a00b51f0cbb448f3a3badc72112b6e55efe20ebaaed9", "artifacts": { "decision-scores.json": "5db2afdef8c68461ce07c395a03b99154170c26d12e0d556277dbd95d14641d6", "humaneval-scores.json": "a45dbf9e8403733fb88107c814b42cf4692de594008781e26d282bdc3976689b", "summary.json": "f5242cac82b7bf5cff012759578278843b3de90a555e32e5c5228c8031444c16", "applied-summary.json": "2e22c62ec1ca3f134f8866ba2348e687bab7b51138ea4938f1b3ec84bf74d93a", "local-scoring-manifest.json": "1dcac9c637d84e251d0be8d5500c8ddf595ce1ef28a0110157c49a0e2d4800e9" } }, { "run_id": "20261010-021726-v2.1-qwen38-hauhaucs-uncensored-q2-k-p", "archive_sha256": "30f2f535e5f0fae14d99204e0c561f147a75dd809cf969788ca3e9bcd82e9a3c", "artifacts": { "decision-scores.json": "e2bd6a3796d706c409e8ce75359e7533c0333931c244c4ce4aaeb452ae65e73f", "humaneval-scores.json": "787662e243eb9e48009c3e3b9c8b0ad46b2fcbda7a36666a4eff9701813f6dbc", "summary.json": "6aa776d4a2371d7c4a1d075b2b59955812eac38c1e4641d2cd48406e2bb40d67", "applied-summary.json": "026894fc8ff3be01df52164d04128e07d5c50d7de4a32f7a4a8d4af889bd4c4e", "local-scoring-manifest.json": "c7366bbd7be0b2dbdb4f5c2c03fcb2403ec176f648d0f6f9d20ce00dea064429" } }, { "run_id": "20261010-030000-v2.1-qwen38-hauhaucs-uncensored-iq3-xs", "archive_sha256": "b64e5a3b8fccb087e095c167b00540b3a5183adcf5f9570009a110f5f7693dd7", "artifacts": { "decision-scores.json": "a9a73b57f0656752d5cf5ae77d3bc4e0cf00529db9432fa0f83aedce30e66119", "humaneval-scores.json": "1c43d51097c9c67277428c6acdf0ad837961707b178e4f0b7f9765059164ee60", "summary.json": "16911beea49e6452b64736e9741a0fc39d50b1b6cfad660dfaf17c7652fc1e42", "applied-summary.json": "4d8d07c71db071383b03bf993246db43c09acec2d8d8bbe1f3eb7e2054d7cff3", "local-scoring-manifest.json": "356b78469f34d1f4179d8a939e1df7e81117967344580dc7c38d6bf2dcf87d8e" } } ] }