Release calibrated Image 2.1 Turbo NVFP4 transformer, accelerated SM120 runtime, quality evidence and real-time demo

.gitattributes

@@ -33,3 +33,78 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+comparisons/00.jpg filter=lfs diff=lfs merge=lfs -text
+comparisons/01.jpg filter=lfs diff=lfs merge=lfs -text
+comparisons/02.jpg filter=lfs diff=lfs merge=lfs -text
+comparisons/03.jpg filter=lfs diff=lfs merge=lfs -text
+comparisons/04.jpg filter=lfs diff=lfs merge=lfs -text
+comparisons/05.jpg filter=lfs diff=lfs merge=lfs -text
+comparisons/06.jpg filter=lfs diff=lfs merge=lfs -text
+comparisons/07.jpg filter=lfs diff=lfs merge=lfs -text
+comparisons/08.jpg filter=lfs diff=lfs merge=lfs -text
+comparisons/09.jpg filter=lfs diff=lfs merge=lfs -text
+comparisons/10.jpg filter=lfs diff=lfs merge=lfs -text
+comparisons/11.jpg filter=lfs diff=lfs merge=lfs -text
+comparisons/12.jpg filter=lfs diff=lfs merge=lfs -text
+comparisons/13.jpg filter=lfs diff=lfs merge=lfs -text
+comparisons/14.jpg filter=lfs diff=lfs merge=lfs -text
+comparisons/15.jpg filter=lfs diff=lfs merge=lfs -text
+comparisons/edit-0.jpg filter=lfs diff=lfs merge=lfs -text
+comparisons/edit-1.jpg filter=lfs diff=lfs merge=lfs -text
+demo/image-001.png filter=lfs diff=lfs merge=lfs -text
+demo/image-002.png filter=lfs diff=lfs merge=lfs -text
+demo/image-003.png filter=lfs diff=lfs merge=lfs -text
+demo/image-004.png filter=lfs diff=lfs merge=lfs -text
+demo/image-005.png filter=lfs diff=lfs merge=lfs -text
+demo/image-006.png filter=lfs diff=lfs merge=lfs -text
+demo/image-007.png filter=lfs diff=lfs merge=lfs -text
+demo/image-008.png filter=lfs diff=lfs merge=lfs -text
+demo/image-009.png filter=lfs diff=lfs merge=lfs -text
+demo/image-010.png filter=lfs diff=lfs merge=lfs -text
+demo/image-011.png filter=lfs diff=lfs merge=lfs -text
+demo/image-012.png filter=lfs diff=lfs merge=lfs -text
+demo/image-013.png filter=lfs diff=lfs merge=lfs -text
+demo/image-014.png filter=lfs diff=lfs merge=lfs -text
+demo/image-015.png filter=lfs diff=lfs merge=lfs -text
+demo/image-016.png filter=lfs diff=lfs merge=lfs -text
+demo/image-017.png filter=lfs diff=lfs merge=lfs -text
+demo/image-018.png filter=lfs diff=lfs merge=lfs -text
+demo/image-019.png filter=lfs diff=lfs merge=lfs -text
+demo/image-020.png filter=lfs diff=lfs merge=lfs -text
+demo/image-021.png filter=lfs diff=lfs merge=lfs -text
+demo/image-022.png filter=lfs diff=lfs merge=lfs -text
+demo/image-023.png filter=lfs diff=lfs merge=lfs -text
+demo/image-024.png filter=lfs diff=lfs merge=lfs -text
+demo/image-025.png filter=lfs diff=lfs merge=lfs -text
+demo/image-026.png filter=lfs diff=lfs merge=lfs -text
+demo/image-027.png filter=lfs diff=lfs merge=lfs -text
+demo/image-028.png filter=lfs diff=lfs merge=lfs -text
+demo/image-029.png filter=lfs diff=lfs merge=lfs -text
+demo/image-030.png filter=lfs diff=lfs merge=lfs -text
+demo/image-031.png filter=lfs diff=lfs merge=lfs -text
+demo/initial.png filter=lfs diff=lfs merge=lfs -text
+demo/realtime-30s-nvfp4.mp4 filter=lfs diff=lfs merge=lfs -text
+demo/video-contact-nvfp4.jpg filter=lfs diff=lfs merge=lfs -text
+examples/nvfp4-2048.png filter=lfs diff=lfs merge=lfs -text
+examples/nvfp4-dragon.png filter=lfs diff=lfs merge=lfs -text
+examples/nvfp4-together-rgb.png filter=lfs diff=lfs merge=lfs -text
+examples/ref-dragon-white.png filter=lfs diff=lfs merge=lfs -text
+examples/ref-whale-white.png filter=lfs diff=lfs merge=lfs -text
+samples/00.png filter=lfs diff=lfs merge=lfs -text
+samples/01.png filter=lfs diff=lfs merge=lfs -text
+samples/02.png filter=lfs diff=lfs merge=lfs -text
+samples/03.png filter=lfs diff=lfs merge=lfs -text
+samples/04.png filter=lfs diff=lfs merge=lfs -text
+samples/05.png filter=lfs diff=lfs merge=lfs -text
+samples/06.png filter=lfs diff=lfs merge=lfs -text
+samples/07.png filter=lfs diff=lfs merge=lfs -text
+samples/08.png filter=lfs diff=lfs merge=lfs -text
+samples/09.png filter=lfs diff=lfs merge=lfs -text
+samples/10.png filter=lfs diff=lfs merge=lfs -text
+samples/11.png filter=lfs diff=lfs merge=lfs -text
+samples/12.png filter=lfs diff=lfs merge=lfs -text
+samples/13.png filter=lfs diff=lfs merge=lfs -text
+samples/14.png filter=lfs diff=lfs merge=lfs -text
+samples/15.png filter=lfs diff=lfs merge=lfs -text
+samples/edit-0.png filter=lfs diff=lfs merge=lfs -text
+samples/edit-1.png filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,55 @@
+Qwen RESEARCH LICENSE AGREEMENT
+
+Qwen RESEARCH LICENSE AGREEMENT Release Date: September 20, 2026
+
+By clicking to agree or by using or distributing any portion or element of the Qwen Materials, you will be deemed to have recognized and accepted the content of this Agreement, which is effective immediately.
+
+1. Definitions
+    a. This Qwen RESEARCH LICENSE AGREEMENT (this "Agreement") shall mean the terms and conditions for use, reproduction, distribution and modification of the Materials as defined by this Agreement.
+    b. "We" (or "Us") shall mean Hangzhou Tongyi Laboratory Technology Co., Ltd.
+    c. "You" (or "Your") shall mean a natural person or legal entity exercising the rights granted by this Agreement and/or using the Materials for any purpose and in any field of use.
+    d. "Third Parties" shall mean individuals or legal entities that are not under common control with us or you.
+    e. "Qwen" shall mean the large language models, diffusion models, and software and algorithms, consisting of trained model weights, parameters (including optimizer states), machine-learning model code, inference-enabling code, training-enabling code, fine-tuning enabling code and other elements of the foregoing distributed by us.
+    f. "Materials" shall mean, collectively, our proprietary Qwen and Documentation (and any portion thereof) made available under this Agreement.
+    g. "Source" form shall mean the preferred form for making modifications, including but not limited to model source code, documentation source, and configuration files.
+    h. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types.
+    i. "Non-Commercial" shall mean for research or evaluation purposes only.
+
+2. Grant of Rights
+    a. You are granted a non-exclusive, worldwide, non-transferable and royalty-free limited license under our intellectual property or other rights owned by us embodied in the Materials to use, reproduce, distribute, copy, create derivative works of, and make modifications to the Materials FOR NON-COMMERCIAL PURPOSES ONLY. 
+    b. You shall not use the Materials for any commercial purpose without obtaining a separate commercial license from us. If you wish to use the Materials commercially, you shall request a license from us at model-business@notice.qwencloud.com.
+
+3. Redistribution
+Subject to Section 2 (Grant of Rights), you may distribute copies or make the Materials, or derivative works thereof, available as part of a product or service that contains any of them, with or without modifications, and in Source or Object form, provided that you meet the following conditions:
+    a. You shall give any other recipients of the Materials or derivative works a copy of this Agreement;
+    b. You shall cause any modified files to carry prominent notices stating that you changed the files;
+    c. You shall retain in all copies of the Materials that you distribute the following attribution notices within a "Notice" text file distributed as a part of such copies: "Qwen is licensed under the Qwen RESEARCH LICENSE AGREEMENT, Copyright (c) 2026 Hangzhou Tongyi Laboratory Technology Co., Ltd. All Rights Reserved."; and
+    d. You may add your own copyright statement to your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of your modifications, or for any such derivative works as a whole, provided your use, reproduction, and distribution of the work otherwise complies with the terms and conditions of this Agreement.
+
+4. Rules of use
+    a. The Materials may be subject to export controls or restrictions in China, the United States or other countries or regions. You shall comply with applicable laws and regulations in your use of the Materials.
+    b. If you use the Materials or any outputs or results therefrom to create, train, fine-tune, or improve an AI model that is distributed or made available, you shall prominently display “Built with Qwen” or “Improved using Qwen” in the related product documentation.
+    c. You shall not use "Qwen" as the primary name or identifier of any derivative works or products; reasonable descriptive use (e.g., "fine-tuned from Qwen Image") is permitted.
+
+5. Intellectual Property
+    a. We retain ownership of all intellectual property rights in and to the Materials and derivatives made by or for us. Conditioned upon compliance with the terms and conditions of this Agreement, with respect to any derivative works and modifications of the Materials that are made by you, you are and will be the owner of such derivative works and modifications.
+    b. No trademark license is granted to use the trade names, trademarks, service marks, or product names of us, except as required to fulfill notice requirements under this Agreement or as required for reasonable and customary use in describing and redistributing the Materials.
+    c. If you commence a lawsuit or other proceedings (including a cross-claim or counterclaim in a lawsuit) against us or any entity alleging that the Materials or any output therefrom, or any part of the foregoing, infringe any intellectual property or other right owned or licensable by you, then all licenses granted to you under this Agreement shall terminate as of the date such lawsuit or other proceeding is commenced or brought.
+
+6. Disclaimer of Warranty and Limitation of Liability
+    a. We are not obligated to support, update, provide training for, or develop any further version of the Qwen Materials or to grant any license thereto.
+    b. THE MATERIALS ARE PROVIDED "AS IS" WITHOUT ANY EXPRESS OR IMPLIED WARRANTY OF ANY KIND INCLUDING WARRANTIES OF MERCHANTABILITY, NONINFRINGEMENT, OR FITNESS FOR A PARTICULAR PURPOSE. WE MAKE NO WARRANTY AND ASSUME NO RESPONSIBILITY FOR THE SAFETY OR STABILITY OF THE MATERIALS AND ANY OUTPUT THEREFROM.
+    c. IN NO EVENT SHALL WE BE LIABLE TO YOU FOR ANY DAMAGES, INCLUDING, BUT NOT LIMITED TO ANY DIRECT, OR INDIRECT, SPECIAL OR CONSEQUENTIAL DAMAGES ARISING FROM YOUR USE OR INABILITY TO USE THE MATERIALS OR ANY OUTPUT OF IT, NO MATTER HOW IT’S CAUSED.
+    d. You will defend, indemnify and hold harmless us from and against any claim by any third party arising out of or related to your use or distribution of the Materials.
+
+7. Survival and Termination.
+    a. The term of this Agreement shall commence upon your acceptance of this Agreement or access to the Materials and will continue in full force and effect until terminated in accordance with the terms and conditions herein.
+    b. We may terminate this Agreement if you breach any of the terms or conditions of this Agreement. Upon termination of this Agreement, you must delete and cease use of the Materials. Sections 6 and 8 shall survive the termination of this Agreement.
+
+8. Governing Law and Jurisdiction.
+    a. This Agreement and any dispute arising out of or relating to it will be governed by the laws of China, without regard to conflict of law principles, and the UN Convention on Contracts for the International Sale of Goods does not apply to this Agreement.
+    b. The People's Courts in Hangzhou City shall have exclusive jurisdiction over any dispute arising out of this Agreement.
+
+9. Other Terms and Conditions.
+    a. Any arrangements, understandings, or agreements regarding the Material not stated herein are separate from and independent of the terms and conditions of this Agreement. You shall request a separate license from us, if you use the Materials in ways not expressly agreed to in this Agreement. 
+    b. We shall not be bound by any additional or different terms or conditions communicated by you unless expressly agreed.

Notice

@@ -0,0 +1,4 @@
+Qwen is licensed under the Qwen RESEARCH LICENSE AGREEMENT, Copyright (c) 2026 Hangzhou Tongyi Laboratory Technology Co., Ltd. All Rights Reserved.
+
+Built with Qwen.
+ProCreations modified the Qwen-Image-2.1-Turbo transformer weights through calibrated NVFP4 quantization on 2026-10-09. The custom runtime, acceleration and calibration/evaluation scripts were added by ProCreations. Non-commercial research and evaluation only under the included upstream license.

README.md

@@ -0,0 +1,128 @@
+---
+license: other
+license_name: qwen-research
+license_link: LICENSE
+base_model: Qwen/Qwen-Image-2.1-Turbo
+base_model_relation: quantized
+pipeline_tag: text-to-image
+library_name: diffusers
+tags:
+- diffusers
+- nvfp4
+- fp4
+- svdquant
+- gptq
+- image-generation
+- image-editing
+- blackwell
+- calibrated
+- turbo
+---
+
+# Image 2.1 Turbo Calibrated NVFP4
+
+**Built with Qwen.** An independently calibrated NVFP4 W4A4 quantization (with BF16 rank-128 corrections) of the image transformer from [Qwen-Image-2.1-Turbo](https://huggingface.co/Qwen/Qwen-Image-2.1-Turbo), with an accelerated runtime. It generates **1024 × 1024 images in 0.96 s end to end** on an RTX PRO 6000 Blackwell. Research/evaluation only under the included **Qwen Research License**. This is not an official Qwen release.
+
+This repository contains the quantized transformer (4.87 GB), an explicit loader and CLI, the full calibration/evaluation source, paired comparisons against BF16 Turbo, raw benchmark and profiler evidence, and a 30-second real-time demonstration. The BF16 text encoder, VAE, processor and the checkpoint's 8-step schedule load from the pinned upstream revision `d65dbc9a7e8f6b5479e33dee6030eaab2a906509`.
+
+## Default: 7 of 8 steps in NVFP4, step 1 in calibrated FP8
+
+By default, **step 1 of the 8-step schedule runs the calibrated FP8 transformer** from [Image-2.1-Turbo-Calibrated-FP8](https://huggingface.co/ProCreations/Image-2.1-Turbo-Calibrated-FP8). That step also fills the prompt/reference KV cache. **Steps 2–8 run NVFP4.** A few-step model fixes its composition early, so this one higher-precision step brings NVFP4 much closer to BF16 (LPIPS 0.123 → 0.086, latent cosine 0.966 → 0.983) at the same speed. FP8's cheaper prefix pass offsets its slower GEMMs. The FP8 transformer (7.26 GB) downloads once from its pinned revision and stays resident, which adds about 7 GB of VRAM. `--fp8-first-steps 0` gives pure NVFP4; `--fp8-first-steps 2` is closer still (LPIPS 0.063–0.071) at about 1.0 s.
+
+## Speed
+
+RTX PRO 6000 Blackwell (SM120, 96 GB), batch 1, all 8 saved-schedule steps, CFG 1, prefix KV cache.
+
+| Runtime | 1024 × 1024 | 2048 × 2048 |
+|---|---:|---:|
+| BF16 Turbo (upstream Diffusers, eager) | 2.05 s | 11.45 s |
+| NVFP4, eager, pure | 1.31 s | 8.40 s |
+| NVFP4 + FP8 step 1, compiled, BF16 attention | 1.03 s | 6.80 s |
+| Pure NVFP4, compiled + SageAttention2 | 0.96 s | 5.33 s |
+| **Release default** (NVFP4 + FP8 step 1, compiled + SageAttention2) | **0.96 s** | **5.38 s** |
+
+CUDA-synchronized end-to-end wall time: prompt encoding, all 8 transformer passes and VAE decode. Excluded: model load (about 4.4 s from local NVMe), first-use compilation in a fresh process (about 11 s at 1024, 15 s at 2048; once per resolution), and PNG writing. Five measured runs at 1024 and three at 2048 (five for the rows measured at both sizes in one process), after one warmup. Release default at 1024: mean 0.960 s, min 0.958 s. Stage breakdown: encoder 16 ms, step 1 (FP8) 138 ms, cached NVFP4 steps 101 ms each, VAE 93 ms. Peak GPU allocation: 33.7 GB (1024) and 42.8 GB (2048) with the FP8 step-1 model resident; 26.4 GB and 35.5 GB for pure NVFP4. Raw numbers: [reports/benchmarks](reports/benchmarks/). With the 2× faster GEMMs, attention is the largest single cost at 2048 (16,384 image tokens).
+
+## Quality against BF16 Turbo
+
+18 held-out cases (16 text-to-image, 2 edits) that were not used for calibration, with matched seeds and resolutions (four at 2048², two non-square, the rest 1024²). Images are composited over white and resized to 512 px for LPIPS (AlexNet) and SSIM. Final latents are compared at full resolution.
+
+| Runtime | Mean LPIPS ↓ | Mean SSIM ↑ | Mean PSNR | Latent cosine |
+|---|---:|---:|---:|---:|
+| **Release default** (FP8 step 1 + NVFP4) | **0.086** | **0.915** | **25.1 dB** | **0.9827** |
+| Pure NVFP4, compiled + Sage | 0.123 | 0.887 | 23.5 dB | 0.9660 |
+| Pure NVFP4, eager | 0.134 | 0.878 | 23.1 dB | 0.9612 |
+| *FP8 release, for reference* | *0.055* | *0.942* | *28.6 dB* | *0.9891* |
+
+All 18 pairs were reviewed visually, along with all 31 images in the demo video. The English and Chinese titles stay correct, transparent RGBA output and both edits work, and fur, feathers, glass and hands look natural. No artifacts were observed. Outputs are **not** identical to BF16: differences show up as composition, pose or identity changes rather than lost detail. With pure NVFP4, a "young man at a pottery wheel" became a different young man and a gouache harbour got a new layout; the FP8 first step restores both. The largest default-mode pair is a macro beetle (LPIPS 0.24, same subject with a shifted fern). SageAttention measured within run-to-run noise (paired runs 0.084–0.089 with Sage vs 0.082–0.087 without). See [comparisons](comparisons/) (BF16 | release default | pure NVFP4 | FP8 release) and [reports/quality_metrics.json](reports/quality_metrics.json). All trials are in [reports/trials](reports/trials/). Keeping every `attn.to_out.0` in FP8 was tested too, but only improved LPIPS 0.134 → 0.130, so it was not adopted. This is a finite fidelity check, not a human-preference study.
+
+## Precision and runtime
+
+* **224 projections** (q/k/v/out and SwiGLU gate/proj/out in all 32 blocks) run as **NVFP4 W4A4**, which works like this:
+  * **Weights:** a per-layer SmoothQuant factor is folded into the weights, a BF16 **rank-128** SVD correction is removed from the smoothed weight, and the residual is packed as E2M1 with E4M3 block scales over 16 elements plus an FP32 global scale.
+  * **Activations:** smoothed and quantized per call with a **dynamic tensor-wide range**, recomputed every call because static ranges clip rare outliers, plus dynamic per-16 block scales.
+  * **GEMM:** native FlashInfer 0.7.1 SM120 CuTe-DSL block-scaled FP4 kernel, with the low-rank BF16 branch fused into the FP32 accumulator epilogue.
+
+  Conditioning, input/output projections, norms, the text encoder and the VAE stay BF16.
+* **Shared low-rank GEMM:** projections that read the same input (q/k/v, gate/proj) compute their rank-128 down-projections in one concatenated GEMM.
+* **Compiled decode:** cached-prefix blocks run under `torch.compile` (dynamic shapes, emulated BF16 cast boundaries).
+* **Split prefill:** step 1's prompt/reference prefix runs alone and fills the KV cache; the image tokens then take the compiled cached path. Prefix tokens never attend to image tokens, so the attention structure is unchanged. The prefix and the image tokens each get their own dynamic activation range.
+* **VAE:** single-frame decode skips Wan's unused video frame cache (bit-identical) and is compiled: 172 → 93 ms at 1024.
+* **Attention:** `--attention sage` uses [SageAttention2](https://github.com/thu-ml/SageAttention) (INT8 QK, FP8 PV, FP32 accumulation) in the cached passes; `--attention bf16` keeps native BF16 SDPA. The default `auto` uses Sage when installed.
+* Not used: step skipping, feature caching, reduced resolution, distillation or LoRA.
+
+Kernel evidence for one 1024² default generation, from the profiler: **1,568 native SM120 block-scaled FP4 GEMM launches** (224 × 7 NVFP4 steps); 448 FP8 GEMMs for step 1 (prefix + target passes); 256 SageAttention `qk_int_sv_f8_attn_kernel` launches. One cached NVFP4 step takes 99 ms of GPU time: FP4 GEMMs 52%, the SageAttention kernel 14%, everything else small fused kernels. See [reports/kernel_evidence.json](reports/kernel_evidence.json).
+
+This is a **custom Diffusers/FlashInfer format** that requires the included loader. It is not a drop-in Transformers, ComfyUI, Nunchaku, vLLM or TensorRT checkpoint, and needs a GPU with SM120 block-scaled FP4 (RTX PRO 6000 / RTX 50-series Blackwell; only SM120 was tested). Do not cast the loaded transformer. Tested with Torch 2.14.0+cu130, Diffusers 0.41.0, Transformers 5.17.0, flashinfer-python 0.7.1 and driver 615.71.09 ([environment](reports/environment.json)).
+
+## Calibration
+
+* **Trajectories:** 64 deterministic BF16 Turbo trajectories on the checkpoint's 8-step schedule: 56 generations and 8 edits, 15 at 2048², 47 at 1024², and two 16:9/9:16. They cover photographs, portraits, textures, illustration, six languages, typography, transparency and compositions.
+* **Activation rows:** 4 token rows per layer at **every** step, 2,048 rows per layer: 512 fit and 512 disjoint diagnostic rows.
+* **Second moments:** full-token input second moments come from all 3.62 M tokens of the same trajectories.
+* **Search per layer:** five smoothing exponents × three residual-weight rounding methods: plain FlashInfer quantizer, activation-weighted 15-point block-scale search, and **GPTQ** error feedback (natural column order, so block scales stay contiguous; per-group scale search). Every candidate is scored with the actual native W4A4 kernel and dynamic range.
+* **Result:** GPTQ won for all 224 layers. Mean diagnostic output NRMSE is **5.03%** (max 10.7%, mid-block `attn.to_out.0`). At the same smoothing, plain rounding averaged 6.40% and block-scale search 5.97% fit NRMSE. The September Image 2.1 (40-step) NVFP4 release was at 5.80%.
+
+Everything is in [calibration_search.json](reports/calibration_search.json), [quantization_summary.json](reports/quantization_summary.json) and [calibration_manifest.json](reports/calibration_manifest.json). This is post-training calibration only.
+
+## Run
+
+Linux, Python 3.12, an SM120 Blackwell GPU and a CUDA 13 driver.
+
+```bash
+hf download ProCreations/Image-2.1-Turbo-Calibrated-NVFP4 --local-dir Image-2.1-Turbo-Calibrated-NVFP4
+cd Image-2.1-Turbo-Calibrated-NVFP4
+python -m venv .venv && source .venv/bin/activate
+pip install torch==2.14.0 torchvision==0.29.0 --index-url https://download.pytorch.org/whl/cu130
+pip install -r requirements.txt
+# optional, faster attention (see requirements.txt): SageAttention 2.2.0 built from source
+python generate.py --prompt "A kingfisher above a forest stream, wildlife photography" --width 1024 --height 1024 --output image.png
+```
+
+The first run downloads the step-1 FP8 transformer from `ProCreations/Image-2.1-Turbo-Calibrated-FP8` at its pinned revision. Use `--fp8-quant /path/to/transformer` for a local copy, or `--fp8-first-steps 0` for pure NVFP4 with no FP8 download. The 8-step schedule comes from the checkpoint. Defaults are 2048 × 2048; use `--width/--height` (multiples of 16; upstream presets 2048², 2400×1792, 1792×2400, 2528×1696, 1696×2528, 2752×1536, 1536×2752). `--warmup` reports first-use cost separately. `--prompts-json list.json` serves many prompts from one loaded pipeline; `--cache-prompts` reuses exact encoder outputs for repeated text-only prompts (0.93 s per 1024² image on a cache hit). `--eager` disables acceleration. `--base /path/to/Qwen-Image-2.1-Turbo` uses a local upstream snapshot.
+
+Library use: `load_pipeline(base, "transformer")` from `nvfp4_runtime.py`, then `accelerate_pipeline(pipe, attention="sage")` from `acceleration.py`, then optionally `enable_fp8_first_steps(pipe, fp8_dir, 1, accelerate=True, attention="sage")`.
+
+### Transparent PNGs, editing and multiple references
+
+```bash
+python generate.py --prompt "A cute mint-green baby dragon mascot, full body" --transparent --width 1024 --height 1024 --output dragon.png
+python generate.py --image input.png --prompt "Change the jacket to blue, keep everything else" --width 1024 --height 1024 --output edited.png
+python generate.py --image examples/ref-dragon-white.png examples/ref-whale-white.png \
+  --prompt "Image 1 shows a mint-green baby dragon. Image 2 shows a blue cartoon whale. Create one new picture with the dragon standing next to the whale, both fully visible." \
+  --width 1024 --height 1024 --output together.png
+```
+
+`--transparent` adds a transparency instruction and keeps the native alpha channel; no background removal is applied ([examples/nvfp4-dragon.png](examples/nvfp4-dragon.png), 74% transparent pixels). References keep command-line order. Transparent RGBA references with a short "place X next to Y" prompt lost the second character in BF16 Turbo as well, so this is upstream behaviour. White-background references with an explicit description worked in BF16, FP8 and NVFP4 ([examples/nvfp4-together-rgb.png](examples/nvfp4-together-rgb.png)).
+
+## Real-time video
+
+[demo/realtime-30s-nvfp4.mp4](demo/realtime-30s-nvfp4.mp4): 30 s, 1024², 30 fps, image only, release default runtime. One completed warmup image is shown at t = 0; after that, each change happens when an actual generation finishes, with all waits kept at 1× speed. The video shows **31 new images in 30 seconds** (mean 0.963 s each, including the Python loop). Maximum capture lag is 2.2 ms, and every sampled decoded frame matches the logged image (min PSNR 34.6 dB). See [verification](demo/video-verification-nvfp4.json), [timestamps](demo/capture_receipt-nvfp4.json) and the [contact sheet](demo/video-contact-nvfp4.jpg). The prompts are disjoint from calibration and evaluation.
+
+## Reproduce
+
+`source/` holds the experiment scripts in run order: `verify_base.py`, `calibrate.py`, `hessians.py`, `quantize_nvfp4.py` (`--shard i/n` workers, then one assembly run; needs `weight_quant.py`, `gptq.py` and `dynamic_scale.py`), `evaluate.py`, `hybrid.py` (step-hybrid trial), `assemble_mixed.py` (FP8 `to_out` trial), `metrics.py`, `speedlab.py`, `benchmark.py`, `video.py`, `verify_video.py`. `common.py` holds the workstation paths. Calibration activations and second moments (about 30 GB) are regenerated locally and are not needed for inference. Unit checks: `test_gptq.py` (the GPTQ packer reproduces the reference FP4 quantizer bit-exactly when H = I, and the native kernel decodes it), `test_vae.py` and `test_split.py`.
+
+## License and modifications
+
+The original model is under the [Qwen Research License](LICENSE): non-commercial research and evaluation, subject to its terms. Keep [Notice](Notice) and the license when redistributing. The modified weights carry a modification notice in their safetensors metadata and in `transformer/quantization_config.json`. ProCreations, 2026-10-09.

acceleration.py

@@ -0,0 +1,242 @@
+"""Acceleration for Image 2.1 Turbo Calibrated FP8 / NVFP4. Built with Qwen.
+
+Defaults keep the checkpoint's full 8-step schedule, calibrated quantized GEMMs with FP32
+accumulation and BF16 attention. No step skipping, feature caching or reduced resolution.
+
+* Cached-prefix decode blocks are compiled (dynamic shapes, emulated BF16 cast boundaries).
+* NVFP4 projections that read the same input (q/k/v, gate/proj) compute their BF16 low-rank
+  down-projections with one concatenated GEMM instead of three/two (same arithmetic per column).
+* The first (KV-cache prefill) step is split: the prompt/reference prefix runs alone, filling the
+  cache, then the target image tokens take the same compiled cached path as later steps. Prefix
+  tokens never attend to target tokens (causal_condition), so this is the same computation.
+* The single-image VAE decode skips the unused video frame cache and is compiled (cuDNN autotuned).
+* Optional attention='sage' uses SageAttention2 (INT8 QK, FP8 PV, FP32 accumulation) in the
+  cached steps; this is an additional approximation and is off unless requested.
+"""
+import math
+import types
+import torch
+from diffusers.models.attention_dispatch import dispatch_attention_fn
+from diffusers.models.modeling_outputs import Transformer2DModelOutput
+from diffusers.models.transformers.transformer_qwenimage21 import (
+    QwenImage21AttnProcessor, QwenImage21TransformerBlock, _IMG_TOKENS_PER_SLOT,
+    _qwenimage21_prefix_segments, _qwenimage21_prepare_qkv,
+)
+
+_ORIGINAL_BLOCK_FORWARD = QwenImage21TransformerBlock.forward
+
+
+@torch.library.custom_op('image21_turbo::sage_attention', mutates_args=())
+def sage_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
+    from sageattention import sageattn_qk_int8_pv_fp8_cuda
+    return sageattn_qk_int8_pv_fp8_cuda(q, k, v, tensor_layout='NHD', is_causal=False,
+                                        qk_quant_gran='per_warp', pv_accum_dtype='fp32+fp32')
+
+
+@sage_attention.register_fake
+def _sage_fake(q, k, v):
+    return torch.empty_like(q)
+
+
+def _real_rope(x, frequencies):
+    paired = x.float().unflatten(-1, (-1, 2))
+    cosine = frequencies.real[None, :, None, :]
+    sine = frequencies.imag[None, :, None, :]
+    return torch.stack((
+        paired[..., 0] * cosine - paired[..., 1] * sine,
+        paired[..., 0] * sine + paired[..., 1] * cosine,
+    ), dim=-1).flatten(-2).to(x.dtype)
+
+
+def _projections(modules, x, shared_down):
+    """Apply projections that share input x. NVFP4 layers reuse one concatenated down GEMM."""
+    if shared_down is None:
+        return [m(x) for m in modules]
+    shape = x.shape[:-1]
+    flat = x.reshape(-1, x.shape[-1]).contiguous()
+    d = flat @ shared_down
+    outputs, start = [], 0
+    for m in modules:
+        r = m.down.shape[1]
+        outputs.append(m.forward_given(flat, d[:, start:start + r]).reshape(*shape, m.out_features))
+        start += r
+    return outputs
+
+
+class NativeAttentionProcessor(QwenImage21AttnProcessor):
+    def __init__(self, attention='bf16'):
+        super().__init__()
+        self.attention = attention
+
+    def __call__(self, attn, hidden_states, attention_mask=None, rotary_emb=None,
+                 layer_cache=None, kv_cache_mode=None, cache_write_slice=None,
+                 segments=None, key_valid=None):
+        if kv_cache_mode == 'prefix':
+            return self._prefix(attn, hidden_states, rotary_emb, layer_cache, cache_write_slice, segments, key_valid)
+        if kv_cache_mode != 'cached' or attention_mask is not None:
+            return super().__call__(attn, hidden_states, attention_mask, rotary_emb,
+                                    layer_cache, kv_cache_mode, cache_write_slice,
+                                    segments, key_valid)
+        query, key, value = _projections((attn.to_q, attn.to_k, attn.to_v), hidden_states,
+                                         getattr(attn, '_image21_shared_down', None))
+        query = query.unflatten(-1, (attn.heads, -1))
+        key = key.unflatten(-1, (attn.heads, -1))
+        value = value.unflatten(-1, (attn.heads, -1))
+        query = attn.norm_q(query).to(value.dtype)
+        key = attn.norm_k(key).to(value.dtype)
+        if rotary_emb is not None:
+            query = _real_rope(query, rotary_emb)
+            key = _real_rope(key, rotary_emb)
+        cached_key, cached_value = layer_cache.get()
+        key = torch.cat((cached_key, key), dim=1)
+        value = torch.cat((cached_value, value), dim=1)
+        if self.attention == 'sage':
+            output = sage_attention(query.contiguous(), key, value)
+        else:
+            output = torch.nn.functional.scaled_dot_product_attention(
+                query.transpose(1, 2), key.transpose(1, 2), value.transpose(1, 2)
+            ).transpose(1, 2)
+        return attn.to_out[1](attn.to_out[0](output.flatten(2, 3)))
+
+
+    def _prefix(self, attn, hidden_states, rotary_emb, layer_cache, cache_write_slice, segments, key_valid):
+        # Upstream prefill attention for the prefix segments only (writes the prefix K/V cache).
+        query, key, value, _ = _qwenimage21_prepare_qkv(attn, hidden_states, rotary_emb, layer_cache, 'extract', cache_write_slice)
+        outputs = []
+        for start, end, is_text in segments:
+            seg_mask = None
+            if is_text:
+                seg_len = end - start
+                seg_mask = torch.cat([
+                    torch.ones(seg_len, start, dtype=torch.bool, device=query.device),
+                    torch.tril(torch.ones(seg_len, seg_len, dtype=torch.bool, device=query.device)),
+                ], dim=1)[None, None]
+            if key_valid is not None:
+                seg_key_valid = key_valid[:, None, None, :end]
+                seg_mask = seg_key_valid if seg_mask is None else (seg_mask & seg_key_valid)
+            outputs.append(dispatch_attention_fn(query[:, start:end], key[:, :end], value[:, :end], attn_mask=seg_mask,
+                                                 dropout_p=0.0, backend=None, parallel_config=self._parallel_config))
+        hidden_states = torch.cat(outputs, dim=1).flatten(2, 3).type_as(query)
+        return attn.to_out[1](attn.to_out[0](hidden_states))
+
+
+def _split_prefill_forward(self, hidden_states, encoder_hidden_states, timestep, img_shapes, img_mask,
+                           encoder_hidden_states_mask=None, attention_kwargs=None, kv_cache=None,
+                           kv_cache_mode=None, return_dict=True):
+    if kv_cache_mode != 'extract' or kv_cache is None or attention_kwargs or not self.config.causal_condition:
+        return self._image21_original_forward(
+            hidden_states=hidden_states, encoder_hidden_states=encoder_hidden_states, timestep=timestep,
+            img_shapes=img_shapes, img_mask=img_mask, encoder_hidden_states_mask=encoder_hidden_states_mask,
+            attention_kwargs=attention_kwargs, kv_cache=kv_cache, kv_cache_mode=kv_cache_mode, return_dict=return_dict)
+    # Same preparation as the upstream forward.
+    batch_size = hidden_states.shape[0]
+    hidden_states = self.img_in(hidden_states)
+    encoder_hidden_states = self.txt_in(encoder_hidden_states)
+    repeats = torch.where(img_mask, _IMG_TOKENS_PER_SLOT, 1)[0]
+    image_pad_mask = torch.repeat_interleave(img_mask[0], repeats)
+    target_tokens = math.prod(img_shapes[0][-1])
+    joint = torch.cat([encoder_hidden_states,
+                       encoder_hidden_states.new_zeros(batch_size, target_tokens // 4, encoder_hidden_states.shape[2])], dim=1)
+    joint = joint.repeat_interleave(repeats, dim=1)
+    joint[:, image_pad_mask] = hidden_states
+    rotary_emb = self.pos_embed(img_shapes[0], image_pad_mask, device=hidden_states.device)
+    image_ids, target_token_mask = self.build_token_metadata(image_pad_mask, img_shapes[0])
+    timestep = timestep.to(hidden_states.dtype)
+    timestep = torch.cat([timestep, timestep.new_zeros(1)], dim=0)
+    temb = self.time_text_embed(timestep, hidden_states)
+    modulation = self.modulation(temb)
+    joint_key_valid = None
+    if encoder_hidden_states_mask is not None:
+        joint_key_valid = torch.ones(batch_size, image_pad_mask.shape[0], dtype=torch.bool, device=hidden_states.device)
+        text_positions = (~image_pad_mask).nonzero(as_tuple=True)[0]
+        vlm_text_positions = ~img_mask[0][: encoder_hidden_states_mask.shape[1]]
+        joint_key_valid[:, text_positions] = encoder_hidden_states_mask.bool()[:, vlm_text_positions]
+    prefix_len = int((~target_token_mask).sum())
+    # 1) prefix alone: block-causal prefill attention over the prompt/reference tokens; fills the cache.
+    segments = _qwenimage21_prefix_segments(image_ids, prefix_len)
+    prefix = joint[:, :prefix_len]
+    prefix_valid = None if joint_key_valid is None else joint_key_valid[:, :prefix_len]
+    for index, block in enumerate(self.transformer_blocks):
+        prefix = block(hidden_states=prefix, modulation=modulation, rotary_emb=rotary_emb[:prefix_len],
+                       attention_mask=None, target_token_mask=target_token_mask[:prefix_len],
+                       layer_cache=kv_cache.get_layer(index), kv_cache_mode='prefix',
+                       cache_write_slice=slice(0, prefix_len), segments=segments, key_valid=prefix_valid)
+    # 2) target tokens: identical attention to the joint prefill (all prefix keys + own block), compiled path.
+    target = joint[:, prefix_len:]
+    target_mask = target_token_mask[prefix_len:]
+    attention_mask = None if joint_key_valid is None else joint_key_valid[:, None, None, :]
+    for index, block in enumerate(self.transformer_blocks):
+        target = block(hidden_states=target, modulation=modulation, rotary_emb=rotary_emb[prefix_len:],
+                       attention_mask=attention_mask, target_token_mask=target_mask,
+                       layer_cache=kv_cache.get_layer(index), kv_cache_mode='cached',
+                       cache_write_slice=None, segments=None, key_valid=None)
+    output = self.proj_out(self.norm_out(target, temb, target_mask))
+    return (output,) if not return_dict else Transformer2DModelOutput(sample=output)
+
+
+def _swiglu_forward(self, hidden_states):
+    gate, proj = _projections((self.gate_layer, self.proj), hidden_states,
+                              getattr(self, '_image21_shared_down', None))
+    return self.out(self.activation_fn(gate) * proj)
+
+
+def _decode_block(self, hidden_states, modulation, rotary_emb=None,
+                  attention_mask=None, target_token_mask=None, layer_cache=None,
+                  kv_cache_mode=None, cache_write_slice=None, segments=None,
+                  key_valid=None):
+    # Cached decode contains only target image tokens. The final t=0 modulation
+    # row belongs to the prefix, which was already evaluated during prefill.
+    if kv_cache_mode == 'cached':
+        modulation = modulation[:-1]
+        target_token_mask = None
+    return _ORIGINAL_BLOCK_FORWARD(
+        self, hidden_states, modulation, rotary_emb, attention_mask,
+        target_token_mask, layer_cache, kv_cache_mode, cache_write_slice,
+        segments, key_valid,
+    )
+
+
+def _dispatch_block(self, **kwargs):
+    if kwargs.get('kv_cache_mode') == 'cached':
+        return self._image21_compiled(**kwargs)
+    return _ORIGINAL_BLOCK_FORWARD(self, **kwargs)
+
+
+def _share_down(owner, modules):
+    if all(hasattr(m, 'forward_given') for m in modules):
+        owner.register_buffer('_image21_shared_down', torch.cat([m.down for m in modules], dim=1).contiguous(),
+                              persistent=False)
+
+
+def accelerate_transformer(transformer, attention='bf16', share_lowrank=True, split_prefill=True):
+    if getattr(transformer, '_image21_accelerated', False):
+        return transformer
+    if split_prefill:
+        transformer._image21_original_forward = transformer.forward
+        transformer.forward = types.MethodType(_split_prefill_forward, transformer)
+    torch._dynamo.config.recompile_limit = max(torch._dynamo.config.recompile_limit, 64)
+    for block in transformer.transformer_blocks:
+        if share_lowrank:
+            _share_down(block.attn, (block.attn.to_q, block.attn.to_k, block.attn.to_v))
+            _share_down(block.img_mlp, (block.img_mlp.gate_layer, block.img_mlp.proj))
+            block.img_mlp.forward = types.MethodType(_swiglu_forward, block.img_mlp)
+        block.attn.set_processor(NativeAttentionProcessor(attention))
+        block._image21_compiled = torch.compile(
+            types.MethodType(_decode_block, block), fullgraph=True, dynamic=True,
+            options={'emulate_precision_casts': True},
+        )
+        block.forward = types.MethodType(_dispatch_block, block)
+    transformer._image21_accelerated = True
+    return transformer
+
+
+def accelerate_pipeline(pipe, attention='bf16', compile_vae=True, share_lowrank=True, split_prefill=True):
+    """Enable once after loading. First-use compilation is excluded from warm timings."""
+    if getattr(pipe, '_image21_accelerated', False):
+        return pipe
+    accelerate_transformer(pipe.transformer, attention, share_lowrank, split_prefill)
+    torch.backends.cudnn.benchmark = True
+    from vae_fast import enable_fast_decode
+    enable_fast_decode(pipe.vae, compile=compile_vae)
+    pipe._image21_accelerated = True
+    return pipe