Image-Text-to-Text
Transformers
Safetensors
Chinese
English
paddleocr_vl
table-recognition
paddleocr-vl
verl
sft
dapo
conversational
custom_code
Instructions to use CheneyWong/paddleocr-vl16-sft800-rl-repro with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use CheneyWong/paddleocr-vl16-sft800-rl-repro with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="CheneyWong/paddleocr-vl16-sft800-rl-repro", trust_remote_code=True) messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoProcessor, AutoModelForMultimodalLM processor = AutoProcessor.from_pretrained("CheneyWong/paddleocr-vl16-sft800-rl-repro", trust_remote_code=True) model = AutoModelForMultimodalLM.from_pretrained("CheneyWong/paddleocr-vl16-sft800-rl-repro", trust_remote_code=True, device_map="auto") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] inputs = processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use CheneyWong/paddleocr-vl16-sft800-rl-repro with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "CheneyWong/paddleocr-vl16-sft800-rl-repro" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "CheneyWong/paddleocr-vl16-sft800-rl-repro", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker
docker model run hf.co/CheneyWong/paddleocr-vl16-sft800-rl-repro
- SGLang
How to use CheneyWong/paddleocr-vl16-sft800-rl-repro with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "CheneyWong/paddleocr-vl16-sft800-rl-repro" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "CheneyWong/paddleocr-vl16-sft800-rl-repro", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "CheneyWong/paddleocr-vl16-sft800-rl-repro" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "CheneyWong/paddleocr-vl16-sft800-rl-repro", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }' - Docker Model Runner
How to use CheneyWong/paddleocr-vl16-sft800-rl-repro with Docker Model Runner:
docker model run hf.co/CheneyWong/paddleocr-vl16-sft800-rl-repro
Add files using upload-large-folder tool
Browse files- MANIFEST.json +0 -0
- README.md +66 -0
- UPLOAD_STATUS.json +6 -0
- added_tokens.json +1021 -0
- chat_template.jinja +43 -0
- config.json +76 -0
- configuration_paddleocr_vl.py +191 -0
- docs/CODEX_GPT.md +78 -0
- docs/ENVIRONMENT.md +87 -0
- docs/EVALUATION.md +67 -0
- docs/PROJECT_CONTEXT.md +58 -0
- docs/REWARD_AND_VERL.md +84 -0
- environment/layout-installed.json +105 -0
- environment/layout-installed.txt +100 -0
- environment/main-installed.json +329 -0
- environment/main-installed.txt +324 -0
- environment/runtime-pinned.txt +256 -0
- export-report.json +35 -0
- generation_config.json +7 -0
- image_processing_paddleocr_vl.py +570 -0
- licenses/verl/LICENSE +202 -0
- modeling_paddleocr_vl.py +0 -0
- preprocessor_config.json +29 -0
- processing_paddleocr_vl.py +293 -0
- processor_config.json +6 -0
- repro/vendor/verl/workers/engine/torchtitan/utils.py +359 -0
- repro/vendor/verl/workers/engine/veomni/transformer_impl.py +989 -0
- repro/vendor/verl/workers/engine/veomni/utils.py +430 -0
- repro/vendor/verl/workers/rollout/naive/__init__.py +17 -0
- repro/vendor/verl/workers/rollout/naive/naive_rollout.py +120 -0
- repro/vendor/verl/workers/rollout/sglang_rollout/__init__.py +12 -0
- repro/vendor/verl/workers/rollout/sglang_rollout/delta_loader.py +228 -0
- repro/vendor/verl/workers/rollout/sglang_rollout/http_server_engine.py +990 -0
- repro/vendor/verl/workers/rollout/sglang_rollout/sglang_pd_replica.py +229 -0
- repro/vendor/verl/workers/rollout/sglang_rollout/sglang_rollout.py +474 -0
- repro/vendor/verl/workers/rollout/sglang_rollout/utils.py +235 -0
- repro/vendor/verl/workers/rollout/trtllm_rollout/__init__.py +13 -0
- repro/vendor/verl/workers/rollout/trtllm_rollout/trtllm_async_server.py +679 -0
- repro/vendor/verl/workers/rollout/trtllm_rollout/trtllm_rollout.py +544 -0
- repro/vendor/verl/workers/rollout/trtllm_rollout/trtllm_worker_extension.py +198 -0
- repro/vendor/verl/workers/rollout/vllm_rollout/__init__.py +42 -0
- repro/vendor/verl/workers/rollout/vllm_rollout/bucketed_weight_transfer.py +355 -0
- repro/vendor/verl/workers/rollout/vllm_rollout/delta_weight_transfer.py +347 -0
- repro/vendor/verl/workers/rollout/vllm_rollout/utils.py +573 -0
- repro/vendor/verl/workers/rollout/vllm_rollout/vllm_async_server.py +1487 -0
- repro/vendor/verl/workers/rollout/vllm_rollout/vllm_pd_replica.py +305 -0
- repro/vendor/verl/workers/rollout/vllm_rollout/vllm_rollout.py +341 -0
- repro/vendor/verl/workers/rollout/vllm_rollout/weight_update_utils.py +65 -0
- special_tokens_map.json +58 -0
- tokenizer_config.json +0 -0
MANIFEST.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
README.md
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
library_name: transformers
|
| 3 |
+
base_model: PaddlePaddle/PaddleOCR-VL-1.6
|
| 4 |
+
tags:
|
| 5 |
+
- table-recognition
|
| 6 |
+
- paddleocr-vl
|
| 7 |
+
- verl
|
| 8 |
+
- sft
|
| 9 |
+
- dapo
|
| 10 |
+
language:
|
| 11 |
+
- zh
|
| 12 |
+
- en
|
| 13 |
+
---
|
| 14 |
+
|
| 15 |
+
# PaddleOCR-VL-1.6 SFT800:权重与实验复现包
|
| 16 |
+
|
| 17 |
+
这是 CheneyWong 的表格识别实验快照,整理日期 **2026-11-16 UTC**。根目录为已选定的 **SFT step800** 完整 HF 权重;不是本轮 RL 的最终模型。仓库按所有者要求公开,便于另一台机器继续迭代。
|
| 18 |
+
|
| 19 |
+
**第一次使用请看 [迁移与环境安装](docs/ENVIRONMENT.md)**,再看 [前情提要](docs/PROJECT_CONTEXT.md)、[reward 与 VeRL 参数](docs/REWARD_AND_VERL.md)、[crop/fullpage 评测](docs/EVALUATION.md)、[Codex 与 GPT 配置](docs/CODEX_GPT.md)。[CPU 验证记录](docs/VALIDATION.md)说明已检查的范围。
|
| 20 |
+
|
| 21 |
+
| 内容 | 位置 |
|
| 22 |
+
|---|---|
|
| 23 |
+
| SFT800 权重、Tokenizer、Processor | 仓库根目录 |
|
| 24 |
+
| 当前 VeRL 运行源码快照 | `repro/vendor/verl/` |
|
| 25 |
+
| Paddle 适配、reward、DAPO / 类 DAPO 训练配置 | `repro/workspace/train/` |
|
| 26 |
+
| 修订后的评测脚本、公式归一化、布局缓存及 GT 覆盖规则 | `repro/workspace/eval/` |
|
| 27 |
+
| 原始 SFT 数据,16,091 条 | `repro/workspace/train/selected-formula-v3/sft-train.parquet` |
|
| 28 |
+
| 当前两种 RL 共用数据,2,499 条 | `repro/workspace/train/adaptive-rl-v2/train.parquet` |
|
| 29 |
+
| 独立 heldout,82 条 | `repro/workspace/train/opd-rl-pilot/data/val.parquet` |
|
| 30 |
+
| crop 评测,173,141 条 / 32 shards | `repro/workspace/datasets/table-recognition-eval-173k/` |
|
| 31 |
+
| fullpage 评测,2,864 页 | `repro/workspace/datasets/table-recognition-fullpage-eval/` |
|
| 32 |
+
| teacher 专项评测原始集合、排除规则、参考分数 | `repro/workspace/eval/teacher-benchmark-v5/` |
|
| 33 |
+
| PP-DocLayoutV3 权重 | `assets/PP-DocLayoutV3/` |
|
| 34 |
+
| 环境记录、逐文件校验 | `environment/`、`MANIFEST.json` |
|
| 35 |
+
|
| 36 |
+
## 下载
|
| 37 |
+
|
| 38 |
+
```bash
|
| 39 |
+
python -m pip install huggingface_hub
|
| 40 |
+
hf download CheneyWong/paddleocr-vl16-sft800-rl-repro \
|
| 41 |
+
--local-dir /workspace/paddle-sft800-repro --max-workers 16
|
| 42 |
+
python /workspace/paddle-sft800-repro/scripts/verify_bundle.py
|
| 43 |
+
```
|
| 44 |
+
|
| 45 |
+
完整快照约 **16.7 GB(十进制)**;不重复存放由原评测集派生的短/长 crop 分片。只需要权重时:
|
| 46 |
+
|
| 47 |
+
```bash
|
| 48 |
+
hf download CheneyWong/paddleocr-vl16-sft800-rl-repro \
|
| 49 |
+
--local-dir /workspace/sft800 \
|
| 50 |
+
--include '*.safetensors' --include '*.json' --include '*.model' \
|
| 51 |
+
--include '*.jinja' --include '*.py' \
|
| 52 |
+
--exclude 'repro/*' --exclude 'assets/*' --exclude 'scripts/*' \
|
| 53 |
+
--exclude 'environment/*' --exclude 'examples/*'
|
| 54 |
+
```
|
| 55 |
+
|
| 56 |
+
新机器采用与原实验相同的 `/workspace/train`、`/workspace/eval`、`/workspace/datasets` 和 `/venv/main` 路径。安装脚本遇到已有目录会拒绝覆盖;不要直接在正在训练的机器执行迁移安装。
|
| 57 |
+
|
| 58 |
+
## 边界
|
| 59 |
+
|
| 60 |
+
- 本包保存 SFT800 的推理/继续微调权重,不包含其 step800 优化器、调度器和 RNG;从它开始新 RL 可复现初始化,不能据此逐位恢复当时 SFT 的中间训练状态。
|
| 61 |
+
- 运行代码包含本项目的 Paddle/VeRL 适配和 checkpoint 空间保护。当前安装环境存在上游依赖声明冲突,安装说明明确记录,不能直接用最新版包替换。
|
| 62 |
+
- 部分训练与专项评测标签来自原数据或机器伪标签,并非全部人工金标。比较模型时必须冻结同一版图片、GT、备用 GT 和布局修正。
|
| 63 |
+
- `licenses/` 及 vendored 源码中保留原许可;上游模型与各数据源仍遵循各自使用条款。本仓库未对全部混合数据重新授予统一许可证。
|
| 64 |
+
- 不含 API/HF 密钥、登录文件、原始 Codex 会话、Qwen 大模型权重及当前 RL checkpoint。生成脚本的示例不会自动调用付费 API。
|
| 65 |
+
|
| 66 |
+
原模型:[PaddlePaddle/PaddleOCR-VL-1.6](https://huggingface.co/PaddlePaddle/PaddleOCR-VL-1.6)。实际文件范围以 `MANIFEST.json` 为准,远程上传完成情况见 `UPLOAD_STATUS.json`。
|
UPLOAD_STATUS.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"phase": "uploading_code_and_documentation",
|
| 3 |
+
"files": 771,
|
| 4 |
+
"bytes": 16722546282,
|
| 5 |
+
"updated_unix": 1794846864.847531
|
| 6 |
+
}
|
added_tokens.json
ADDED
|
@@ -0,0 +1,1021 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"<ecel>": 101308,
|
| 3 |
+
"<fcel>": 101309,
|
| 4 |
+
"<lcel>": 101311,
|
| 5 |
+
"<nl>": 101313,
|
| 6 |
+
"<ucel>": 101312,
|
| 7 |
+
"<xcel>": 101310,
|
| 8 |
+
"<|AUDIO_PLACEHOLDER|>": 100296,
|
| 9 |
+
"<|CROP_COL_SEP|>": 101301,
|
| 10 |
+
"<|CROP_ROW_SEP|>": 101302,
|
| 11 |
+
"<|IMAGE_END|>": 101306,
|
| 12 |
+
"<|IMAGE_PLACEHOLDER|>": 100295,
|
| 13 |
+
"<|IMAGE_SEP|>": 101303,
|
| 14 |
+
"<|IMAGE_START|>": 101305,
|
| 15 |
+
"<|LOC_0|>": 100297,
|
| 16 |
+
"<|LOC_1000|>": 101297,
|
| 17 |
+
"<|LOC_100|>": 100397,
|
| 18 |
+
"<|LOC_101|>": 100398,
|
| 19 |
+
"<|LOC_102|>": 100399,
|
| 20 |
+
"<|LOC_103|>": 100400,
|
| 21 |
+
"<|LOC_104|>": 100401,
|
| 22 |
+
"<|LOC_105|>": 100402,
|
| 23 |
+
"<|LOC_106|>": 100403,
|
| 24 |
+
"<|LOC_107|>": 100404,
|
| 25 |
+
"<|LOC_108|>": 100405,
|
| 26 |
+
"<|LOC_109|>": 100406,
|
| 27 |
+
"<|LOC_10|>": 100307,
|
| 28 |
+
"<|LOC_110|>": 100407,
|
| 29 |
+
"<|LOC_111|>": 100408,
|
| 30 |
+
"<|LOC_112|>": 100409,
|
| 31 |
+
"<|LOC_113|>": 100410,
|
| 32 |
+
"<|LOC_114|>": 100411,
|
| 33 |
+
"<|LOC_115|>": 100412,
|
| 34 |
+
"<|LOC_116|>": 100413,
|
| 35 |
+
"<|LOC_117|>": 100414,
|
| 36 |
+
"<|LOC_118|>": 100415,
|
| 37 |
+
"<|LOC_119|>": 100416,
|
| 38 |
+
"<|LOC_11|>": 100308,
|
| 39 |
+
"<|LOC_120|>": 100417,
|
| 40 |
+
"<|LOC_121|>": 100418,
|
| 41 |
+
"<|LOC_122|>": 100419,
|
| 42 |
+
"<|LOC_123|>": 100420,
|
| 43 |
+
"<|LOC_124|>": 100421,
|
| 44 |
+
"<|LOC_125|>": 100422,
|
| 45 |
+
"<|LOC_126|>": 100423,
|
| 46 |
+
"<|LOC_127|>": 100424,
|
| 47 |
+
"<|LOC_128|>": 100425,
|
| 48 |
+
"<|LOC_129|>": 100426,
|
| 49 |
+
"<|LOC_12|>": 100309,
|
| 50 |
+
"<|LOC_130|>": 100427,
|
| 51 |
+
"<|LOC_131|>": 100428,
|
| 52 |
+
"<|LOC_132|>": 100429,
|
| 53 |
+
"<|LOC_133|>": 100430,
|
| 54 |
+
"<|LOC_134|>": 100431,
|
| 55 |
+
"<|LOC_135|>": 100432,
|
| 56 |
+
"<|LOC_136|>": 100433,
|
| 57 |
+
"<|LOC_137|>": 100434,
|
| 58 |
+
"<|LOC_138|>": 100435,
|
| 59 |
+
"<|LOC_139|>": 100436,
|
| 60 |
+
"<|LOC_13|>": 100310,
|
| 61 |
+
"<|LOC_140|>": 100437,
|
| 62 |
+
"<|LOC_141|>": 100438,
|
| 63 |
+
"<|LOC_142|>": 100439,
|
| 64 |
+
"<|LOC_143|>": 100440,
|
| 65 |
+
"<|LOC_144|>": 100441,
|
| 66 |
+
"<|LOC_145|>": 100442,
|
| 67 |
+
"<|LOC_146|>": 100443,
|
| 68 |
+
"<|LOC_147|>": 100444,
|
| 69 |
+
"<|LOC_148|>": 100445,
|
| 70 |
+
"<|LOC_149|>": 100446,
|
| 71 |
+
"<|LOC_14|>": 100311,
|
| 72 |
+
"<|LOC_150|>": 100447,
|
| 73 |
+
"<|LOC_151|>": 100448,
|
| 74 |
+
"<|LOC_152|>": 100449,
|
| 75 |
+
"<|LOC_153|>": 100450,
|
| 76 |
+
"<|LOC_154|>": 100451,
|
| 77 |
+
"<|LOC_155|>": 100452,
|
| 78 |
+
"<|LOC_156|>": 100453,
|
| 79 |
+
"<|LOC_157|>": 100454,
|
| 80 |
+
"<|LOC_158|>": 100455,
|
| 81 |
+
"<|LOC_159|>": 100456,
|
| 82 |
+
"<|LOC_15|>": 100312,
|
| 83 |
+
"<|LOC_160|>": 100457,
|
| 84 |
+
"<|LOC_161|>": 100458,
|
| 85 |
+
"<|LOC_162|>": 100459,
|
| 86 |
+
"<|LOC_163|>": 100460,
|
| 87 |
+
"<|LOC_164|>": 100461,
|
| 88 |
+
"<|LOC_165|>": 100462,
|
| 89 |
+
"<|LOC_166|>": 100463,
|
| 90 |
+
"<|LOC_167|>": 100464,
|
| 91 |
+
"<|LOC_168|>": 100465,
|
| 92 |
+
"<|LOC_169|>": 100466,
|
| 93 |
+
"<|LOC_16|>": 100313,
|
| 94 |
+
"<|LOC_170|>": 100467,
|
| 95 |
+
"<|LOC_171|>": 100468,
|
| 96 |
+
"<|LOC_172|>": 100469,
|
| 97 |
+
"<|LOC_173|>": 100470,
|
| 98 |
+
"<|LOC_174|>": 100471,
|
| 99 |
+
"<|LOC_175|>": 100472,
|
| 100 |
+
"<|LOC_176|>": 100473,
|
| 101 |
+
"<|LOC_177|>": 100474,
|
| 102 |
+
"<|LOC_178|>": 100475,
|
| 103 |
+
"<|LOC_179|>": 100476,
|
| 104 |
+
"<|LOC_17|>": 100314,
|
| 105 |
+
"<|LOC_180|>": 100477,
|
| 106 |
+
"<|LOC_181|>": 100478,
|
| 107 |
+
"<|LOC_182|>": 100479,
|
| 108 |
+
"<|LOC_183|>": 100480,
|
| 109 |
+
"<|LOC_184|>": 100481,
|
| 110 |
+
"<|LOC_185|>": 100482,
|
| 111 |
+
"<|LOC_186|>": 100483,
|
| 112 |
+
"<|LOC_187|>": 100484,
|
| 113 |
+
"<|LOC_188|>": 100485,
|
| 114 |
+
"<|LOC_189|>": 100486,
|
| 115 |
+
"<|LOC_18|>": 100315,
|
| 116 |
+
"<|LOC_190|>": 100487,
|
| 117 |
+
"<|LOC_191|>": 100488,
|
| 118 |
+
"<|LOC_192|>": 100489,
|
| 119 |
+
"<|LOC_193|>": 100490,
|
| 120 |
+
"<|LOC_194|>": 100491,
|
| 121 |
+
"<|LOC_195|>": 100492,
|
| 122 |
+
"<|LOC_196|>": 100493,
|
| 123 |
+
"<|LOC_197|>": 100494,
|
| 124 |
+
"<|LOC_198|>": 100495,
|
| 125 |
+
"<|LOC_199|>": 100496,
|
| 126 |
+
"<|LOC_19|>": 100316,
|
| 127 |
+
"<|LOC_1|>": 100298,
|
| 128 |
+
"<|LOC_200|>": 100497,
|
| 129 |
+
"<|LOC_201|>": 100498,
|
| 130 |
+
"<|LOC_202|>": 100499,
|
| 131 |
+
"<|LOC_203|>": 100500,
|
| 132 |
+
"<|LOC_204|>": 100501,
|
| 133 |
+
"<|LOC_205|>": 100502,
|
| 134 |
+
"<|LOC_206|>": 100503,
|
| 135 |
+
"<|LOC_207|>": 100504,
|
| 136 |
+
"<|LOC_208|>": 100505,
|
| 137 |
+
"<|LOC_209|>": 100506,
|
| 138 |
+
"<|LOC_20|>": 100317,
|
| 139 |
+
"<|LOC_210|>": 100507,
|
| 140 |
+
"<|LOC_211|>": 100508,
|
| 141 |
+
"<|LOC_212|>": 100509,
|
| 142 |
+
"<|LOC_213|>": 100510,
|
| 143 |
+
"<|LOC_214|>": 100511,
|
| 144 |
+
"<|LOC_215|>": 100512,
|
| 145 |
+
"<|LOC_216|>": 100513,
|
| 146 |
+
"<|LOC_217|>": 100514,
|
| 147 |
+
"<|LOC_218|>": 100515,
|
| 148 |
+
"<|LOC_219|>": 100516,
|
| 149 |
+
"<|LOC_21|>": 100318,
|
| 150 |
+
"<|LOC_220|>": 100517,
|
| 151 |
+
"<|LOC_221|>": 100518,
|
| 152 |
+
"<|LOC_222|>": 100519,
|
| 153 |
+
"<|LOC_223|>": 100520,
|
| 154 |
+
"<|LOC_224|>": 100521,
|
| 155 |
+
"<|LOC_225|>": 100522,
|
| 156 |
+
"<|LOC_226|>": 100523,
|
| 157 |
+
"<|LOC_227|>": 100524,
|
| 158 |
+
"<|LOC_228|>": 100525,
|
| 159 |
+
"<|LOC_229|>": 100526,
|
| 160 |
+
"<|LOC_22|>": 100319,
|
| 161 |
+
"<|LOC_230|>": 100527,
|
| 162 |
+
"<|LOC_231|>": 100528,
|
| 163 |
+
"<|LOC_232|>": 100529,
|
| 164 |
+
"<|LOC_233|>": 100530,
|
| 165 |
+
"<|LOC_234|>": 100531,
|
| 166 |
+
"<|LOC_235|>": 100532,
|
| 167 |
+
"<|LOC_236|>": 100533,
|
| 168 |
+
"<|LOC_237|>": 100534,
|
| 169 |
+
"<|LOC_238|>": 100535,
|
| 170 |
+
"<|LOC_239|>": 100536,
|
| 171 |
+
"<|LOC_23|>": 100320,
|
| 172 |
+
"<|LOC_240|>": 100537,
|
| 173 |
+
"<|LOC_241|>": 100538,
|
| 174 |
+
"<|LOC_242|>": 100539,
|
| 175 |
+
"<|LOC_243|>": 100540,
|
| 176 |
+
"<|LOC_244|>": 100541,
|
| 177 |
+
"<|LOC_245|>": 100542,
|
| 178 |
+
"<|LOC_246|>": 100543,
|
| 179 |
+
"<|LOC_247|>": 100544,
|
| 180 |
+
"<|LOC_248|>": 100545,
|
| 181 |
+
"<|LOC_249|>": 100546,
|
| 182 |
+
"<|LOC_24|>": 100321,
|
| 183 |
+
"<|LOC_250|>": 100547,
|
| 184 |
+
"<|LOC_251|>": 100548,
|
| 185 |
+
"<|LOC_252|>": 100549,
|
| 186 |
+
"<|LOC_253|>": 100550,
|
| 187 |
+
"<|LOC_254|>": 100551,
|
| 188 |
+
"<|LOC_255|>": 100552,
|
| 189 |
+
"<|LOC_256|>": 100553,
|
| 190 |
+
"<|LOC_257|>": 100554,
|
| 191 |
+
"<|LOC_258|>": 100555,
|
| 192 |
+
"<|LOC_259|>": 100556,
|
| 193 |
+
"<|LOC_25|>": 100322,
|
| 194 |
+
"<|LOC_260|>": 100557,
|
| 195 |
+
"<|LOC_261|>": 100558,
|
| 196 |
+
"<|LOC_262|>": 100559,
|
| 197 |
+
"<|LOC_263|>": 100560,
|
| 198 |
+
"<|LOC_264|>": 100561,
|
| 199 |
+
"<|LOC_265|>": 100562,
|
| 200 |
+
"<|LOC_266|>": 100563,
|
| 201 |
+
"<|LOC_267|>": 100564,
|
| 202 |
+
"<|LOC_268|>": 100565,
|
| 203 |
+
"<|LOC_269|>": 100566,
|
| 204 |
+
"<|LOC_26|>": 100323,
|
| 205 |
+
"<|LOC_270|>": 100567,
|
| 206 |
+
"<|LOC_271|>": 100568,
|
| 207 |
+
"<|LOC_272|>": 100569,
|
| 208 |
+
"<|LOC_273|>": 100570,
|
| 209 |
+
"<|LOC_274|>": 100571,
|
| 210 |
+
"<|LOC_275|>": 100572,
|
| 211 |
+
"<|LOC_276|>": 100573,
|
| 212 |
+
"<|LOC_277|>": 100574,
|
| 213 |
+
"<|LOC_278|>": 100575,
|
| 214 |
+
"<|LOC_279|>": 100576,
|
| 215 |
+
"<|LOC_27|>": 100324,
|
| 216 |
+
"<|LOC_280|>": 100577,
|
| 217 |
+
"<|LOC_281|>": 100578,
|
| 218 |
+
"<|LOC_282|>": 100579,
|
| 219 |
+
"<|LOC_283|>": 100580,
|
| 220 |
+
"<|LOC_284|>": 100581,
|
| 221 |
+
"<|LOC_285|>": 100582,
|
| 222 |
+
"<|LOC_286|>": 100583,
|
| 223 |
+
"<|LOC_287|>": 100584,
|
| 224 |
+
"<|LOC_288|>": 100585,
|
| 225 |
+
"<|LOC_289|>": 100586,
|
| 226 |
+
"<|LOC_28|>": 100325,
|
| 227 |
+
"<|LOC_290|>": 100587,
|
| 228 |
+
"<|LOC_291|>": 100588,
|
| 229 |
+
"<|LOC_292|>": 100589,
|
| 230 |
+
"<|LOC_293|>": 100590,
|
| 231 |
+
"<|LOC_294|>": 100591,
|
| 232 |
+
"<|LOC_295|>": 100592,
|
| 233 |
+
"<|LOC_296|>": 100593,
|
| 234 |
+
"<|LOC_297|>": 100594,
|
| 235 |
+
"<|LOC_298|>": 100595,
|
| 236 |
+
"<|LOC_299|>": 100596,
|
| 237 |
+
"<|LOC_29|>": 100326,
|
| 238 |
+
"<|LOC_2|>": 100299,
|
| 239 |
+
"<|LOC_300|>": 100597,
|
| 240 |
+
"<|LOC_301|>": 100598,
|
| 241 |
+
"<|LOC_302|>": 100599,
|
| 242 |
+
"<|LOC_303|>": 100600,
|
| 243 |
+
"<|LOC_304|>": 100601,
|
| 244 |
+
"<|LOC_305|>": 100602,
|
| 245 |
+
"<|LOC_306|>": 100603,
|
| 246 |
+
"<|LOC_307|>": 100604,
|
| 247 |
+
"<|LOC_308|>": 100605,
|
| 248 |
+
"<|LOC_309|>": 100606,
|
| 249 |
+
"<|LOC_30|>": 100327,
|
| 250 |
+
"<|LOC_310|>": 100607,
|
| 251 |
+
"<|LOC_311|>": 100608,
|
| 252 |
+
"<|LOC_312|>": 100609,
|
| 253 |
+
"<|LOC_313|>": 100610,
|
| 254 |
+
"<|LOC_314|>": 100611,
|
| 255 |
+
"<|LOC_315|>": 100612,
|
| 256 |
+
"<|LOC_316|>": 100613,
|
| 257 |
+
"<|LOC_317|>": 100614,
|
| 258 |
+
"<|LOC_318|>": 100615,
|
| 259 |
+
"<|LOC_319|>": 100616,
|
| 260 |
+
"<|LOC_31|>": 100328,
|
| 261 |
+
"<|LOC_320|>": 100617,
|
| 262 |
+
"<|LOC_321|>": 100618,
|
| 263 |
+
"<|LOC_322|>": 100619,
|
| 264 |
+
"<|LOC_323|>": 100620,
|
| 265 |
+
"<|LOC_324|>": 100621,
|
| 266 |
+
"<|LOC_325|>": 100622,
|
| 267 |
+
"<|LOC_326|>": 100623,
|
| 268 |
+
"<|LOC_327|>": 100624,
|
| 269 |
+
"<|LOC_328|>": 100625,
|
| 270 |
+
"<|LOC_329|>": 100626,
|
| 271 |
+
"<|LOC_32|>": 100329,
|
| 272 |
+
"<|LOC_330|>": 100627,
|
| 273 |
+
"<|LOC_331|>": 100628,
|
| 274 |
+
"<|LOC_332|>": 100629,
|
| 275 |
+
"<|LOC_333|>": 100630,
|
| 276 |
+
"<|LOC_334|>": 100631,
|
| 277 |
+
"<|LOC_335|>": 100632,
|
| 278 |
+
"<|LOC_336|>": 100633,
|
| 279 |
+
"<|LOC_337|>": 100634,
|
| 280 |
+
"<|LOC_338|>": 100635,
|
| 281 |
+
"<|LOC_339|>": 100636,
|
| 282 |
+
"<|LOC_33|>": 100330,
|
| 283 |
+
"<|LOC_340|>": 100637,
|
| 284 |
+
"<|LOC_341|>": 100638,
|
| 285 |
+
"<|LOC_342|>": 100639,
|
| 286 |
+
"<|LOC_343|>": 100640,
|
| 287 |
+
"<|LOC_344|>": 100641,
|
| 288 |
+
"<|LOC_345|>": 100642,
|
| 289 |
+
"<|LOC_346|>": 100643,
|
| 290 |
+
"<|LOC_347|>": 100644,
|
| 291 |
+
"<|LOC_348|>": 100645,
|
| 292 |
+
"<|LOC_349|>": 100646,
|
| 293 |
+
"<|LOC_34|>": 100331,
|
| 294 |
+
"<|LOC_350|>": 100647,
|
| 295 |
+
"<|LOC_351|>": 100648,
|
| 296 |
+
"<|LOC_352|>": 100649,
|
| 297 |
+
"<|LOC_353|>": 100650,
|
| 298 |
+
"<|LOC_354|>": 100651,
|
| 299 |
+
"<|LOC_355|>": 100652,
|
| 300 |
+
"<|LOC_356|>": 100653,
|
| 301 |
+
"<|LOC_357|>": 100654,
|
| 302 |
+
"<|LOC_358|>": 100655,
|
| 303 |
+
"<|LOC_359|>": 100656,
|
| 304 |
+
"<|LOC_35|>": 100332,
|
| 305 |
+
"<|LOC_360|>": 100657,
|
| 306 |
+
"<|LOC_361|>": 100658,
|
| 307 |
+
"<|LOC_362|>": 100659,
|
| 308 |
+
"<|LOC_363|>": 100660,
|
| 309 |
+
"<|LOC_364|>": 100661,
|
| 310 |
+
"<|LOC_365|>": 100662,
|
| 311 |
+
"<|LOC_366|>": 100663,
|
| 312 |
+
"<|LOC_367|>": 100664,
|
| 313 |
+
"<|LOC_368|>": 100665,
|
| 314 |
+
"<|LOC_369|>": 100666,
|
| 315 |
+
"<|LOC_36|>": 100333,
|
| 316 |
+
"<|LOC_370|>": 100667,
|
| 317 |
+
"<|LOC_371|>": 100668,
|
| 318 |
+
"<|LOC_372|>": 100669,
|
| 319 |
+
"<|LOC_373|>": 100670,
|
| 320 |
+
"<|LOC_374|>": 100671,
|
| 321 |
+
"<|LOC_375|>": 100672,
|
| 322 |
+
"<|LOC_376|>": 100673,
|
| 323 |
+
"<|LOC_377|>": 100674,
|
| 324 |
+
"<|LOC_378|>": 100675,
|
| 325 |
+
"<|LOC_379|>": 100676,
|
| 326 |
+
"<|LOC_37|>": 100334,
|
| 327 |
+
"<|LOC_380|>": 100677,
|
| 328 |
+
"<|LOC_381|>": 100678,
|
| 329 |
+
"<|LOC_382|>": 100679,
|
| 330 |
+
"<|LOC_383|>": 100680,
|
| 331 |
+
"<|LOC_384|>": 100681,
|
| 332 |
+
"<|LOC_385|>": 100682,
|
| 333 |
+
"<|LOC_386|>": 100683,
|
| 334 |
+
"<|LOC_387|>": 100684,
|
| 335 |
+
"<|LOC_388|>": 100685,
|
| 336 |
+
"<|LOC_389|>": 100686,
|
| 337 |
+
"<|LOC_38|>": 100335,
|
| 338 |
+
"<|LOC_390|>": 100687,
|
| 339 |
+
"<|LOC_391|>": 100688,
|
| 340 |
+
"<|LOC_392|>": 100689,
|
| 341 |
+
"<|LOC_393|>": 100690,
|
| 342 |
+
"<|LOC_394|>": 100691,
|
| 343 |
+
"<|LOC_395|>": 100692,
|
| 344 |
+
"<|LOC_396|>": 100693,
|
| 345 |
+
"<|LOC_397|>": 100694,
|
| 346 |
+
"<|LOC_398|>": 100695,
|
| 347 |
+
"<|LOC_399|>": 100696,
|
| 348 |
+
"<|LOC_39|>": 100336,
|
| 349 |
+
"<|LOC_3|>": 100300,
|
| 350 |
+
"<|LOC_400|>": 100697,
|
| 351 |
+
"<|LOC_401|>": 100698,
|
| 352 |
+
"<|LOC_402|>": 100699,
|
| 353 |
+
"<|LOC_403|>": 100700,
|
| 354 |
+
"<|LOC_404|>": 100701,
|
| 355 |
+
"<|LOC_405|>": 100702,
|
| 356 |
+
"<|LOC_406|>": 100703,
|
| 357 |
+
"<|LOC_407|>": 100704,
|
| 358 |
+
"<|LOC_408|>": 100705,
|
| 359 |
+
"<|LOC_409|>": 100706,
|
| 360 |
+
"<|LOC_40|>": 100337,
|
| 361 |
+
"<|LOC_410|>": 100707,
|
| 362 |
+
"<|LOC_411|>": 100708,
|
| 363 |
+
"<|LOC_412|>": 100709,
|
| 364 |
+
"<|LOC_413|>": 100710,
|
| 365 |
+
"<|LOC_414|>": 100711,
|
| 366 |
+
"<|LOC_415|>": 100712,
|
| 367 |
+
"<|LOC_416|>": 100713,
|
| 368 |
+
"<|LOC_417|>": 100714,
|
| 369 |
+
"<|LOC_418|>": 100715,
|
| 370 |
+
"<|LOC_419|>": 100716,
|
| 371 |
+
"<|LOC_41|>": 100338,
|
| 372 |
+
"<|LOC_420|>": 100717,
|
| 373 |
+
"<|LOC_421|>": 100718,
|
| 374 |
+
"<|LOC_422|>": 100719,
|
| 375 |
+
"<|LOC_423|>": 100720,
|
| 376 |
+
"<|LOC_424|>": 100721,
|
| 377 |
+
"<|LOC_425|>": 100722,
|
| 378 |
+
"<|LOC_426|>": 100723,
|
| 379 |
+
"<|LOC_427|>": 100724,
|
| 380 |
+
"<|LOC_428|>": 100725,
|
| 381 |
+
"<|LOC_429|>": 100726,
|
| 382 |
+
"<|LOC_42|>": 100339,
|
| 383 |
+
"<|LOC_430|>": 100727,
|
| 384 |
+
"<|LOC_431|>": 100728,
|
| 385 |
+
"<|LOC_432|>": 100729,
|
| 386 |
+
"<|LOC_433|>": 100730,
|
| 387 |
+
"<|LOC_434|>": 100731,
|
| 388 |
+
"<|LOC_435|>": 100732,
|
| 389 |
+
"<|LOC_436|>": 100733,
|
| 390 |
+
"<|LOC_437|>": 100734,
|
| 391 |
+
"<|LOC_438|>": 100735,
|
| 392 |
+
"<|LOC_439|>": 100736,
|
| 393 |
+
"<|LOC_43|>": 100340,
|
| 394 |
+
"<|LOC_440|>": 100737,
|
| 395 |
+
"<|LOC_441|>": 100738,
|
| 396 |
+
"<|LOC_442|>": 100739,
|
| 397 |
+
"<|LOC_443|>": 100740,
|
| 398 |
+
"<|LOC_444|>": 100741,
|
| 399 |
+
"<|LOC_445|>": 100742,
|
| 400 |
+
"<|LOC_446|>": 100743,
|
| 401 |
+
"<|LOC_447|>": 100744,
|
| 402 |
+
"<|LOC_448|>": 100745,
|
| 403 |
+
"<|LOC_449|>": 100746,
|
| 404 |
+
"<|LOC_44|>": 100341,
|
| 405 |
+
"<|LOC_450|>": 100747,
|
| 406 |
+
"<|LOC_451|>": 100748,
|
| 407 |
+
"<|LOC_452|>": 100749,
|
| 408 |
+
"<|LOC_453|>": 100750,
|
| 409 |
+
"<|LOC_454|>": 100751,
|
| 410 |
+
"<|LOC_455|>": 100752,
|
| 411 |
+
"<|LOC_456|>": 100753,
|
| 412 |
+
"<|LOC_457|>": 100754,
|
| 413 |
+
"<|LOC_458|>": 100755,
|
| 414 |
+
"<|LOC_459|>": 100756,
|
| 415 |
+
"<|LOC_45|>": 100342,
|
| 416 |
+
"<|LOC_460|>": 100757,
|
| 417 |
+
"<|LOC_461|>": 100758,
|
| 418 |
+
"<|LOC_462|>": 100759,
|
| 419 |
+
"<|LOC_463|>": 100760,
|
| 420 |
+
"<|LOC_464|>": 100761,
|
| 421 |
+
"<|LOC_465|>": 100762,
|
| 422 |
+
"<|LOC_466|>": 100763,
|
| 423 |
+
"<|LOC_467|>": 100764,
|
| 424 |
+
"<|LOC_468|>": 100765,
|
| 425 |
+
"<|LOC_469|>": 100766,
|
| 426 |
+
"<|LOC_46|>": 100343,
|
| 427 |
+
"<|LOC_470|>": 100767,
|
| 428 |
+
"<|LOC_471|>": 100768,
|
| 429 |
+
"<|LOC_472|>": 100769,
|
| 430 |
+
"<|LOC_473|>": 100770,
|
| 431 |
+
"<|LOC_474|>": 100771,
|
| 432 |
+
"<|LOC_475|>": 100772,
|
| 433 |
+
"<|LOC_476|>": 100773,
|
| 434 |
+
"<|LOC_477|>": 100774,
|
| 435 |
+
"<|LOC_478|>": 100775,
|
| 436 |
+
"<|LOC_479|>": 100776,
|
| 437 |
+
"<|LOC_47|>": 100344,
|
| 438 |
+
"<|LOC_480|>": 100777,
|
| 439 |
+
"<|LOC_481|>": 100778,
|
| 440 |
+
"<|LOC_482|>": 100779,
|
| 441 |
+
"<|LOC_483|>": 100780,
|
| 442 |
+
"<|LOC_484|>": 100781,
|
| 443 |
+
"<|LOC_485|>": 100782,
|
| 444 |
+
"<|LOC_486|>": 100783,
|
| 445 |
+
"<|LOC_487|>": 100784,
|
| 446 |
+
"<|LOC_488|>": 100785,
|
| 447 |
+
"<|LOC_489|>": 100786,
|
| 448 |
+
"<|LOC_48|>": 100345,
|
| 449 |
+
"<|LOC_490|>": 100787,
|
| 450 |
+
"<|LOC_491|>": 100788,
|
| 451 |
+
"<|LOC_492|>": 100789,
|
| 452 |
+
"<|LOC_493|>": 100790,
|
| 453 |
+
"<|LOC_494|>": 100791,
|
| 454 |
+
"<|LOC_495|>": 100792,
|
| 455 |
+
"<|LOC_496|>": 100793,
|
| 456 |
+
"<|LOC_497|>": 100794,
|
| 457 |
+
"<|LOC_498|>": 100795,
|
| 458 |
+
"<|LOC_499|>": 100796,
|
| 459 |
+
"<|LOC_49|>": 100346,
|
| 460 |
+
"<|LOC_4|>": 100301,
|
| 461 |
+
"<|LOC_500|>": 100797,
|
| 462 |
+
"<|LOC_501|>": 100798,
|
| 463 |
+
"<|LOC_502|>": 100799,
|
| 464 |
+
"<|LOC_503|>": 100800,
|
| 465 |
+
"<|LOC_504|>": 100801,
|
| 466 |
+
"<|LOC_505|>": 100802,
|
| 467 |
+
"<|LOC_506|>": 100803,
|
| 468 |
+
"<|LOC_507|>": 100804,
|
| 469 |
+
"<|LOC_508|>": 100805,
|
| 470 |
+
"<|LOC_509|>": 100806,
|
| 471 |
+
"<|LOC_50|>": 100347,
|
| 472 |
+
"<|LOC_510|>": 100807,
|
| 473 |
+
"<|LOC_511|>": 100808,
|
| 474 |
+
"<|LOC_512|>": 100809,
|
| 475 |
+
"<|LOC_513|>": 100810,
|
| 476 |
+
"<|LOC_514|>": 100811,
|
| 477 |
+
"<|LOC_515|>": 100812,
|
| 478 |
+
"<|LOC_516|>": 100813,
|
| 479 |
+
"<|LOC_517|>": 100814,
|
| 480 |
+
"<|LOC_518|>": 100815,
|
| 481 |
+
"<|LOC_519|>": 100816,
|
| 482 |
+
"<|LOC_51|>": 100348,
|
| 483 |
+
"<|LOC_520|>": 100817,
|
| 484 |
+
"<|LOC_521|>": 100818,
|
| 485 |
+
"<|LOC_522|>": 100819,
|
| 486 |
+
"<|LOC_523|>": 100820,
|
| 487 |
+
"<|LOC_524|>": 100821,
|
| 488 |
+
"<|LOC_525|>": 100822,
|
| 489 |
+
"<|LOC_526|>": 100823,
|
| 490 |
+
"<|LOC_527|>": 100824,
|
| 491 |
+
"<|LOC_528|>": 100825,
|
| 492 |
+
"<|LOC_529|>": 100826,
|
| 493 |
+
"<|LOC_52|>": 100349,
|
| 494 |
+
"<|LOC_530|>": 100827,
|
| 495 |
+
"<|LOC_531|>": 100828,
|
| 496 |
+
"<|LOC_532|>": 100829,
|
| 497 |
+
"<|LOC_533|>": 100830,
|
| 498 |
+
"<|LOC_534|>": 100831,
|
| 499 |
+
"<|LOC_535|>": 100832,
|
| 500 |
+
"<|LOC_536|>": 100833,
|
| 501 |
+
"<|LOC_537|>": 100834,
|
| 502 |
+
"<|LOC_538|>": 100835,
|
| 503 |
+
"<|LOC_539|>": 100836,
|
| 504 |
+
"<|LOC_53|>": 100350,
|
| 505 |
+
"<|LOC_540|>": 100837,
|
| 506 |
+
"<|LOC_541|>": 100838,
|
| 507 |
+
"<|LOC_542|>": 100839,
|
| 508 |
+
"<|LOC_543|>": 100840,
|
| 509 |
+
"<|LOC_544|>": 100841,
|
| 510 |
+
"<|LOC_545|>": 100842,
|
| 511 |
+
"<|LOC_546|>": 100843,
|
| 512 |
+
"<|LOC_547|>": 100844,
|
| 513 |
+
"<|LOC_548|>": 100845,
|
| 514 |
+
"<|LOC_549|>": 100846,
|
| 515 |
+
"<|LOC_54|>": 100351,
|
| 516 |
+
"<|LOC_550|>": 100847,
|
| 517 |
+
"<|LOC_551|>": 100848,
|
| 518 |
+
"<|LOC_552|>": 100849,
|
| 519 |
+
"<|LOC_553|>": 100850,
|
| 520 |
+
"<|LOC_554|>": 100851,
|
| 521 |
+
"<|LOC_555|>": 100852,
|
| 522 |
+
"<|LOC_556|>": 100853,
|
| 523 |
+
"<|LOC_557|>": 100854,
|
| 524 |
+
"<|LOC_558|>": 100855,
|
| 525 |
+
"<|LOC_559|>": 100856,
|
| 526 |
+
"<|LOC_55|>": 100352,
|
| 527 |
+
"<|LOC_560|>": 100857,
|
| 528 |
+
"<|LOC_561|>": 100858,
|
| 529 |
+
"<|LOC_562|>": 100859,
|
| 530 |
+
"<|LOC_563|>": 100860,
|
| 531 |
+
"<|LOC_564|>": 100861,
|
| 532 |
+
"<|LOC_565|>": 100862,
|
| 533 |
+
"<|LOC_566|>": 100863,
|
| 534 |
+
"<|LOC_567|>": 100864,
|
| 535 |
+
"<|LOC_568|>": 100865,
|
| 536 |
+
"<|LOC_569|>": 100866,
|
| 537 |
+
"<|LOC_56|>": 100353,
|
| 538 |
+
"<|LOC_570|>": 100867,
|
| 539 |
+
"<|LOC_571|>": 100868,
|
| 540 |
+
"<|LOC_572|>": 100869,
|
| 541 |
+
"<|LOC_573|>": 100870,
|
| 542 |
+
"<|LOC_574|>": 100871,
|
| 543 |
+
"<|LOC_575|>": 100872,
|
| 544 |
+
"<|LOC_576|>": 100873,
|
| 545 |
+
"<|LOC_577|>": 100874,
|
| 546 |
+
"<|LOC_578|>": 100875,
|
| 547 |
+
"<|LOC_579|>": 100876,
|
| 548 |
+
"<|LOC_57|>": 100354,
|
| 549 |
+
"<|LOC_580|>": 100877,
|
| 550 |
+
"<|LOC_581|>": 100878,
|
| 551 |
+
"<|LOC_582|>": 100879,
|
| 552 |
+
"<|LOC_583|>": 100880,
|
| 553 |
+
"<|LOC_584|>": 100881,
|
| 554 |
+
"<|LOC_585|>": 100882,
|
| 555 |
+
"<|LOC_586|>": 100883,
|
| 556 |
+
"<|LOC_587|>": 100884,
|
| 557 |
+
"<|LOC_588|>": 100885,
|
| 558 |
+
"<|LOC_589|>": 100886,
|
| 559 |
+
"<|LOC_58|>": 100355,
|
| 560 |
+
"<|LOC_590|>": 100887,
|
| 561 |
+
"<|LOC_591|>": 100888,
|
| 562 |
+
"<|LOC_592|>": 100889,
|
| 563 |
+
"<|LOC_593|>": 100890,
|
| 564 |
+
"<|LOC_594|>": 100891,
|
| 565 |
+
"<|LOC_595|>": 100892,
|
| 566 |
+
"<|LOC_596|>": 100893,
|
| 567 |
+
"<|LOC_597|>": 100894,
|
| 568 |
+
"<|LOC_598|>": 100895,
|
| 569 |
+
"<|LOC_599|>": 100896,
|
| 570 |
+
"<|LOC_59|>": 100356,
|
| 571 |
+
"<|LOC_5|>": 100302,
|
| 572 |
+
"<|LOC_600|>": 100897,
|
| 573 |
+
"<|LOC_601|>": 100898,
|
| 574 |
+
"<|LOC_602|>": 100899,
|
| 575 |
+
"<|LOC_603|>": 100900,
|
| 576 |
+
"<|LOC_604|>": 100901,
|
| 577 |
+
"<|LOC_605|>": 100902,
|
| 578 |
+
"<|LOC_606|>": 100903,
|
| 579 |
+
"<|LOC_607|>": 100904,
|
| 580 |
+
"<|LOC_608|>": 100905,
|
| 581 |
+
"<|LOC_609|>": 100906,
|
| 582 |
+
"<|LOC_60|>": 100357,
|
| 583 |
+
"<|LOC_610|>": 100907,
|
| 584 |
+
"<|LOC_611|>": 100908,
|
| 585 |
+
"<|LOC_612|>": 100909,
|
| 586 |
+
"<|LOC_613|>": 100910,
|
| 587 |
+
"<|LOC_614|>": 100911,
|
| 588 |
+
"<|LOC_615|>": 100912,
|
| 589 |
+
"<|LOC_616|>": 100913,
|
| 590 |
+
"<|LOC_617|>": 100914,
|
| 591 |
+
"<|LOC_618|>": 100915,
|
| 592 |
+
"<|LOC_619|>": 100916,
|
| 593 |
+
"<|LOC_61|>": 100358,
|
| 594 |
+
"<|LOC_620|>": 100917,
|
| 595 |
+
"<|LOC_621|>": 100918,
|
| 596 |
+
"<|LOC_622|>": 100919,
|
| 597 |
+
"<|LOC_623|>": 100920,
|
| 598 |
+
"<|LOC_624|>": 100921,
|
| 599 |
+
"<|LOC_625|>": 100922,
|
| 600 |
+
"<|LOC_626|>": 100923,
|
| 601 |
+
"<|LOC_627|>": 100924,
|
| 602 |
+
"<|LOC_628|>": 100925,
|
| 603 |
+
"<|LOC_629|>": 100926,
|
| 604 |
+
"<|LOC_62|>": 100359,
|
| 605 |
+
"<|LOC_630|>": 100927,
|
| 606 |
+
"<|LOC_631|>": 100928,
|
| 607 |
+
"<|LOC_632|>": 100929,
|
| 608 |
+
"<|LOC_633|>": 100930,
|
| 609 |
+
"<|LOC_634|>": 100931,
|
| 610 |
+
"<|LOC_635|>": 100932,
|
| 611 |
+
"<|LOC_636|>": 100933,
|
| 612 |
+
"<|LOC_637|>": 100934,
|
| 613 |
+
"<|LOC_638|>": 100935,
|
| 614 |
+
"<|LOC_639|>": 100936,
|
| 615 |
+
"<|LOC_63|>": 100360,
|
| 616 |
+
"<|LOC_640|>": 100937,
|
| 617 |
+
"<|LOC_641|>": 100938,
|
| 618 |
+
"<|LOC_642|>": 100939,
|
| 619 |
+
"<|LOC_643|>": 100940,
|
| 620 |
+
"<|LOC_644|>": 100941,
|
| 621 |
+
"<|LOC_645|>": 100942,
|
| 622 |
+
"<|LOC_646|>": 100943,
|
| 623 |
+
"<|LOC_647|>": 100944,
|
| 624 |
+
"<|LOC_648|>": 100945,
|
| 625 |
+
"<|LOC_649|>": 100946,
|
| 626 |
+
"<|LOC_64|>": 100361,
|
| 627 |
+
"<|LOC_650|>": 100947,
|
| 628 |
+
"<|LOC_651|>": 100948,
|
| 629 |
+
"<|LOC_652|>": 100949,
|
| 630 |
+
"<|LOC_653|>": 100950,
|
| 631 |
+
"<|LOC_654|>": 100951,
|
| 632 |
+
"<|LOC_655|>": 100952,
|
| 633 |
+
"<|LOC_656|>": 100953,
|
| 634 |
+
"<|LOC_657|>": 100954,
|
| 635 |
+
"<|LOC_658|>": 100955,
|
| 636 |
+
"<|LOC_659|>": 100956,
|
| 637 |
+
"<|LOC_65|>": 100362,
|
| 638 |
+
"<|LOC_660|>": 100957,
|
| 639 |
+
"<|LOC_661|>": 100958,
|
| 640 |
+
"<|LOC_662|>": 100959,
|
| 641 |
+
"<|LOC_663|>": 100960,
|
| 642 |
+
"<|LOC_664|>": 100961,
|
| 643 |
+
"<|LOC_665|>": 100962,
|
| 644 |
+
"<|LOC_666|>": 100963,
|
| 645 |
+
"<|LOC_667|>": 100964,
|
| 646 |
+
"<|LOC_668|>": 100965,
|
| 647 |
+
"<|LOC_669|>": 100966,
|
| 648 |
+
"<|LOC_66|>": 100363,
|
| 649 |
+
"<|LOC_670|>": 100967,
|
| 650 |
+
"<|LOC_671|>": 100968,
|
| 651 |
+
"<|LOC_672|>": 100969,
|
| 652 |
+
"<|LOC_673|>": 100970,
|
| 653 |
+
"<|LOC_674|>": 100971,
|
| 654 |
+
"<|LOC_675|>": 100972,
|
| 655 |
+
"<|LOC_676|>": 100973,
|
| 656 |
+
"<|LOC_677|>": 100974,
|
| 657 |
+
"<|LOC_678|>": 100975,
|
| 658 |
+
"<|LOC_679|>": 100976,
|
| 659 |
+
"<|LOC_67|>": 100364,
|
| 660 |
+
"<|LOC_680|>": 100977,
|
| 661 |
+
"<|LOC_681|>": 100978,
|
| 662 |
+
"<|LOC_682|>": 100979,
|
| 663 |
+
"<|LOC_683|>": 100980,
|
| 664 |
+
"<|LOC_684|>": 100981,
|
| 665 |
+
"<|LOC_685|>": 100982,
|
| 666 |
+
"<|LOC_686|>": 100983,
|
| 667 |
+
"<|LOC_687|>": 100984,
|
| 668 |
+
"<|LOC_688|>": 100985,
|
| 669 |
+
"<|LOC_689|>": 100986,
|
| 670 |
+
"<|LOC_68|>": 100365,
|
| 671 |
+
"<|LOC_690|>": 100987,
|
| 672 |
+
"<|LOC_691|>": 100988,
|
| 673 |
+
"<|LOC_692|>": 100989,
|
| 674 |
+
"<|LOC_693|>": 100990,
|
| 675 |
+
"<|LOC_694|>": 100991,
|
| 676 |
+
"<|LOC_695|>": 100992,
|
| 677 |
+
"<|LOC_696|>": 100993,
|
| 678 |
+
"<|LOC_697|>": 100994,
|
| 679 |
+
"<|LOC_698|>": 100995,
|
| 680 |
+
"<|LOC_699|>": 100996,
|
| 681 |
+
"<|LOC_69|>": 100366,
|
| 682 |
+
"<|LOC_6|>": 100303,
|
| 683 |
+
"<|LOC_700|>": 100997,
|
| 684 |
+
"<|LOC_701|>": 100998,
|
| 685 |
+
"<|LOC_702|>": 100999,
|
| 686 |
+
"<|LOC_703|>": 101000,
|
| 687 |
+
"<|LOC_704|>": 101001,
|
| 688 |
+
"<|LOC_705|>": 101002,
|
| 689 |
+
"<|LOC_706|>": 101003,
|
| 690 |
+
"<|LOC_707|>": 101004,
|
| 691 |
+
"<|LOC_708|>": 101005,
|
| 692 |
+
"<|LOC_709|>": 101006,
|
| 693 |
+
"<|LOC_70|>": 100367,
|
| 694 |
+
"<|LOC_710|>": 101007,
|
| 695 |
+
"<|LOC_711|>": 101008,
|
| 696 |
+
"<|LOC_712|>": 101009,
|
| 697 |
+
"<|LOC_713|>": 101010,
|
| 698 |
+
"<|LOC_714|>": 101011,
|
| 699 |
+
"<|LOC_715|>": 101012,
|
| 700 |
+
"<|LOC_716|>": 101013,
|
| 701 |
+
"<|LOC_717|>": 101014,
|
| 702 |
+
"<|LOC_718|>": 101015,
|
| 703 |
+
"<|LOC_719|>": 101016,
|
| 704 |
+
"<|LOC_71|>": 100368,
|
| 705 |
+
"<|LOC_720|>": 101017,
|
| 706 |
+
"<|LOC_721|>": 101018,
|
| 707 |
+
"<|LOC_722|>": 101019,
|
| 708 |
+
"<|LOC_723|>": 101020,
|
| 709 |
+
"<|LOC_724|>": 101021,
|
| 710 |
+
"<|LOC_725|>": 101022,
|
| 711 |
+
"<|LOC_726|>": 101023,
|
| 712 |
+
"<|LOC_727|>": 101024,
|
| 713 |
+
"<|LOC_728|>": 101025,
|
| 714 |
+
"<|LOC_729|>": 101026,
|
| 715 |
+
"<|LOC_72|>": 100369,
|
| 716 |
+
"<|LOC_730|>": 101027,
|
| 717 |
+
"<|LOC_731|>": 101028,
|
| 718 |
+
"<|LOC_732|>": 101029,
|
| 719 |
+
"<|LOC_733|>": 101030,
|
| 720 |
+
"<|LOC_734|>": 101031,
|
| 721 |
+
"<|LOC_735|>": 101032,
|
| 722 |
+
"<|LOC_736|>": 101033,
|
| 723 |
+
"<|LOC_737|>": 101034,
|
| 724 |
+
"<|LOC_738|>": 101035,
|
| 725 |
+
"<|LOC_739|>": 101036,
|
| 726 |
+
"<|LOC_73|>": 100370,
|
| 727 |
+
"<|LOC_740|>": 101037,
|
| 728 |
+
"<|LOC_741|>": 101038,
|
| 729 |
+
"<|LOC_742|>": 101039,
|
| 730 |
+
"<|LOC_743|>": 101040,
|
| 731 |
+
"<|LOC_744|>": 101041,
|
| 732 |
+
"<|LOC_745|>": 101042,
|
| 733 |
+
"<|LOC_746|>": 101043,
|
| 734 |
+
"<|LOC_747|>": 101044,
|
| 735 |
+
"<|LOC_748|>": 101045,
|
| 736 |
+
"<|LOC_749|>": 101046,
|
| 737 |
+
"<|LOC_74|>": 100371,
|
| 738 |
+
"<|LOC_750|>": 101047,
|
| 739 |
+
"<|LOC_751|>": 101048,
|
| 740 |
+
"<|LOC_752|>": 101049,
|
| 741 |
+
"<|LOC_753|>": 101050,
|
| 742 |
+
"<|LOC_754|>": 101051,
|
| 743 |
+
"<|LOC_755|>": 101052,
|
| 744 |
+
"<|LOC_756|>": 101053,
|
| 745 |
+
"<|LOC_757|>": 101054,
|
| 746 |
+
"<|LOC_758|>": 101055,
|
| 747 |
+
"<|LOC_759|>": 101056,
|
| 748 |
+
"<|LOC_75|>": 100372,
|
| 749 |
+
"<|LOC_760|>": 101057,
|
| 750 |
+
"<|LOC_761|>": 101058,
|
| 751 |
+
"<|LOC_762|>": 101059,
|
| 752 |
+
"<|LOC_763|>": 101060,
|
| 753 |
+
"<|LOC_764|>": 101061,
|
| 754 |
+
"<|LOC_765|>": 101062,
|
| 755 |
+
"<|LOC_766|>": 101063,
|
| 756 |
+
"<|LOC_767|>": 101064,
|
| 757 |
+
"<|LOC_768|>": 101065,
|
| 758 |
+
"<|LOC_769|>": 101066,
|
| 759 |
+
"<|LOC_76|>": 100373,
|
| 760 |
+
"<|LOC_770|>": 101067,
|
| 761 |
+
"<|LOC_771|>": 101068,
|
| 762 |
+
"<|LOC_772|>": 101069,
|
| 763 |
+
"<|LOC_773|>": 101070,
|
| 764 |
+
"<|LOC_774|>": 101071,
|
| 765 |
+
"<|LOC_775|>": 101072,
|
| 766 |
+
"<|LOC_776|>": 101073,
|
| 767 |
+
"<|LOC_777|>": 101074,
|
| 768 |
+
"<|LOC_778|>": 101075,
|
| 769 |
+
"<|LOC_779|>": 101076,
|
| 770 |
+
"<|LOC_77|>": 100374,
|
| 771 |
+
"<|LOC_780|>": 101077,
|
| 772 |
+
"<|LOC_781|>": 101078,
|
| 773 |
+
"<|LOC_782|>": 101079,
|
| 774 |
+
"<|LOC_783|>": 101080,
|
| 775 |
+
"<|LOC_784|>": 101081,
|
| 776 |
+
"<|LOC_785|>": 101082,
|
| 777 |
+
"<|LOC_786|>": 101083,
|
| 778 |
+
"<|LOC_787|>": 101084,
|
| 779 |
+
"<|LOC_788|>": 101085,
|
| 780 |
+
"<|LOC_789|>": 101086,
|
| 781 |
+
"<|LOC_78|>": 100375,
|
| 782 |
+
"<|LOC_790|>": 101087,
|
| 783 |
+
"<|LOC_791|>": 101088,
|
| 784 |
+
"<|LOC_792|>": 101089,
|
| 785 |
+
"<|LOC_793|>": 101090,
|
| 786 |
+
"<|LOC_794|>": 101091,
|
| 787 |
+
"<|LOC_795|>": 101092,
|
| 788 |
+
"<|LOC_796|>": 101093,
|
| 789 |
+
"<|LOC_797|>": 101094,
|
| 790 |
+
"<|LOC_798|>": 101095,
|
| 791 |
+
"<|LOC_799|>": 101096,
|
| 792 |
+
"<|LOC_79|>": 100376,
|
| 793 |
+
"<|LOC_7|>": 100304,
|
| 794 |
+
"<|LOC_800|>": 101097,
|
| 795 |
+
"<|LOC_801|>": 101098,
|
| 796 |
+
"<|LOC_802|>": 101099,
|
| 797 |
+
"<|LOC_803|>": 101100,
|
| 798 |
+
"<|LOC_804|>": 101101,
|
| 799 |
+
"<|LOC_805|>": 101102,
|
| 800 |
+
"<|LOC_806|>": 101103,
|
| 801 |
+
"<|LOC_807|>": 101104,
|
| 802 |
+
"<|LOC_808|>": 101105,
|
| 803 |
+
"<|LOC_809|>": 101106,
|
| 804 |
+
"<|LOC_80|>": 100377,
|
| 805 |
+
"<|LOC_810|>": 101107,
|
| 806 |
+
"<|LOC_811|>": 101108,
|
| 807 |
+
"<|LOC_812|>": 101109,
|
| 808 |
+
"<|LOC_813|>": 101110,
|
| 809 |
+
"<|LOC_814|>": 101111,
|
| 810 |
+
"<|LOC_815|>": 101112,
|
| 811 |
+
"<|LOC_816|>": 101113,
|
| 812 |
+
"<|LOC_817|>": 101114,
|
| 813 |
+
"<|LOC_818|>": 101115,
|
| 814 |
+
"<|LOC_819|>": 101116,
|
| 815 |
+
"<|LOC_81|>": 100378,
|
| 816 |
+
"<|LOC_820|>": 101117,
|
| 817 |
+
"<|LOC_821|>": 101118,
|
| 818 |
+
"<|LOC_822|>": 101119,
|
| 819 |
+
"<|LOC_823|>": 101120,
|
| 820 |
+
"<|LOC_824|>": 101121,
|
| 821 |
+
"<|LOC_825|>": 101122,
|
| 822 |
+
"<|LOC_826|>": 101123,
|
| 823 |
+
"<|LOC_827|>": 101124,
|
| 824 |
+
"<|LOC_828|>": 101125,
|
| 825 |
+
"<|LOC_829|>": 101126,
|
| 826 |
+
"<|LOC_82|>": 100379,
|
| 827 |
+
"<|LOC_830|>": 101127,
|
| 828 |
+
"<|LOC_831|>": 101128,
|
| 829 |
+
"<|LOC_832|>": 101129,
|
| 830 |
+
"<|LOC_833|>": 101130,
|
| 831 |
+
"<|LOC_834|>": 101131,
|
| 832 |
+
"<|LOC_835|>": 101132,
|
| 833 |
+
"<|LOC_836|>": 101133,
|
| 834 |
+
"<|LOC_837|>": 101134,
|
| 835 |
+
"<|LOC_838|>": 101135,
|
| 836 |
+
"<|LOC_839|>": 101136,
|
| 837 |
+
"<|LOC_83|>": 100380,
|
| 838 |
+
"<|LOC_840|>": 101137,
|
| 839 |
+
"<|LOC_841|>": 101138,
|
| 840 |
+
"<|LOC_842|>": 101139,
|
| 841 |
+
"<|LOC_843|>": 101140,
|
| 842 |
+
"<|LOC_844|>": 101141,
|
| 843 |
+
"<|LOC_845|>": 101142,
|
| 844 |
+
"<|LOC_846|>": 101143,
|
| 845 |
+
"<|LOC_847|>": 101144,
|
| 846 |
+
"<|LOC_848|>": 101145,
|
| 847 |
+
"<|LOC_849|>": 101146,
|
| 848 |
+
"<|LOC_84|>": 100381,
|
| 849 |
+
"<|LOC_850|>": 101147,
|
| 850 |
+
"<|LOC_851|>": 101148,
|
| 851 |
+
"<|LOC_852|>": 101149,
|
| 852 |
+
"<|LOC_853|>": 101150,
|
| 853 |
+
"<|LOC_854|>": 101151,
|
| 854 |
+
"<|LOC_855|>": 101152,
|
| 855 |
+
"<|LOC_856|>": 101153,
|
| 856 |
+
"<|LOC_857|>": 101154,
|
| 857 |
+
"<|LOC_858|>": 101155,
|
| 858 |
+
"<|LOC_859|>": 101156,
|
| 859 |
+
"<|LOC_85|>": 100382,
|
| 860 |
+
"<|LOC_860|>": 101157,
|
| 861 |
+
"<|LOC_861|>": 101158,
|
| 862 |
+
"<|LOC_862|>": 101159,
|
| 863 |
+
"<|LOC_863|>": 101160,
|
| 864 |
+
"<|LOC_864|>": 101161,
|
| 865 |
+
"<|LOC_865|>": 101162,
|
| 866 |
+
"<|LOC_866|>": 101163,
|
| 867 |
+
"<|LOC_867|>": 101164,
|
| 868 |
+
"<|LOC_868|>": 101165,
|
| 869 |
+
"<|LOC_869|>": 101166,
|
| 870 |
+
"<|LOC_86|>": 100383,
|
| 871 |
+
"<|LOC_870|>": 101167,
|
| 872 |
+
"<|LOC_871|>": 101168,
|
| 873 |
+
"<|LOC_872|>": 101169,
|
| 874 |
+
"<|LOC_873|>": 101170,
|
| 875 |
+
"<|LOC_874|>": 101171,
|
| 876 |
+
"<|LOC_875|>": 101172,
|
| 877 |
+
"<|LOC_876|>": 101173,
|
| 878 |
+
"<|LOC_877|>": 101174,
|
| 879 |
+
"<|LOC_878|>": 101175,
|
| 880 |
+
"<|LOC_879|>": 101176,
|
| 881 |
+
"<|LOC_87|>": 100384,
|
| 882 |
+
"<|LOC_880|>": 101177,
|
| 883 |
+
"<|LOC_881|>": 101178,
|
| 884 |
+
"<|LOC_882|>": 101179,
|
| 885 |
+
"<|LOC_883|>": 101180,
|
| 886 |
+
"<|LOC_884|>": 101181,
|
| 887 |
+
"<|LOC_885|>": 101182,
|
| 888 |
+
"<|LOC_886|>": 101183,
|
| 889 |
+
"<|LOC_887|>": 101184,
|
| 890 |
+
"<|LOC_888|>": 101185,
|
| 891 |
+
"<|LOC_889|>": 101186,
|
| 892 |
+
"<|LOC_88|>": 100385,
|
| 893 |
+
"<|LOC_890|>": 101187,
|
| 894 |
+
"<|LOC_891|>": 101188,
|
| 895 |
+
"<|LOC_892|>": 101189,
|
| 896 |
+
"<|LOC_893|>": 101190,
|
| 897 |
+
"<|LOC_894|>": 101191,
|
| 898 |
+
"<|LOC_895|>": 101192,
|
| 899 |
+
"<|LOC_896|>": 101193,
|
| 900 |
+
"<|LOC_897|>": 101194,
|
| 901 |
+
"<|LOC_898|>": 101195,
|
| 902 |
+
"<|LOC_899|>": 101196,
|
| 903 |
+
"<|LOC_89|>": 100386,
|
| 904 |
+
"<|LOC_8|>": 100305,
|
| 905 |
+
"<|LOC_900|>": 101197,
|
| 906 |
+
"<|LOC_901|>": 101198,
|
| 907 |
+
"<|LOC_902|>": 101199,
|
| 908 |
+
"<|LOC_903|>": 101200,
|
| 909 |
+
"<|LOC_904|>": 101201,
|
| 910 |
+
"<|LOC_905|>": 101202,
|
| 911 |
+
"<|LOC_906|>": 101203,
|
| 912 |
+
"<|LOC_907|>": 101204,
|
| 913 |
+
"<|LOC_908|>": 101205,
|
| 914 |
+
"<|LOC_909|>": 101206,
|
| 915 |
+
"<|LOC_90|>": 100387,
|
| 916 |
+
"<|LOC_910|>": 101207,
|
| 917 |
+
"<|LOC_911|>": 101208,
|
| 918 |
+
"<|LOC_912|>": 101209,
|
| 919 |
+
"<|LOC_913|>": 101210,
|
| 920 |
+
"<|LOC_914|>": 101211,
|
| 921 |
+
"<|LOC_915|>": 101212,
|
| 922 |
+
"<|LOC_916|>": 101213,
|
| 923 |
+
"<|LOC_917|>": 101214,
|
| 924 |
+
"<|LOC_918|>": 101215,
|
| 925 |
+
"<|LOC_919|>": 101216,
|
| 926 |
+
"<|LOC_91|>": 100388,
|
| 927 |
+
"<|LOC_920|>": 101217,
|
| 928 |
+
"<|LOC_921|>": 101218,
|
| 929 |
+
"<|LOC_922|>": 101219,
|
| 930 |
+
"<|LOC_923|>": 101220,
|
| 931 |
+
"<|LOC_924|>": 101221,
|
| 932 |
+
"<|LOC_925|>": 101222,
|
| 933 |
+
"<|LOC_926|>": 101223,
|
| 934 |
+
"<|LOC_927|>": 101224,
|
| 935 |
+
"<|LOC_928|>": 101225,
|
| 936 |
+
"<|LOC_929|>": 101226,
|
| 937 |
+
"<|LOC_92|>": 100389,
|
| 938 |
+
"<|LOC_930|>": 101227,
|
| 939 |
+
"<|LOC_931|>": 101228,
|
| 940 |
+
"<|LOC_932|>": 101229,
|
| 941 |
+
"<|LOC_933|>": 101230,
|
| 942 |
+
"<|LOC_934|>": 101231,
|
| 943 |
+
"<|LOC_935|>": 101232,
|
| 944 |
+
"<|LOC_936|>": 101233,
|
| 945 |
+
"<|LOC_937|>": 101234,
|
| 946 |
+
"<|LOC_938|>": 101235,
|
| 947 |
+
"<|LOC_939|>": 101236,
|
| 948 |
+
"<|LOC_93|>": 100390,
|
| 949 |
+
"<|LOC_940|>": 101237,
|
| 950 |
+
"<|LOC_941|>": 101238,
|
| 951 |
+
"<|LOC_942|>": 101239,
|
| 952 |
+
"<|LOC_943|>": 101240,
|
| 953 |
+
"<|LOC_944|>": 101241,
|
| 954 |
+
"<|LOC_945|>": 101242,
|
| 955 |
+
"<|LOC_946|>": 101243,
|
| 956 |
+
"<|LOC_947|>": 101244,
|
| 957 |
+
"<|LOC_948|>": 101245,
|
| 958 |
+
"<|LOC_949|>": 101246,
|
| 959 |
+
"<|LOC_94|>": 100391,
|
| 960 |
+
"<|LOC_950|>": 101247,
|
| 961 |
+
"<|LOC_951|>": 101248,
|
| 962 |
+
"<|LOC_952|>": 101249,
|
| 963 |
+
"<|LOC_953|>": 101250,
|
| 964 |
+
"<|LOC_954|>": 101251,
|
| 965 |
+
"<|LOC_955|>": 101252,
|
| 966 |
+
"<|LOC_956|>": 101253,
|
| 967 |
+
"<|LOC_957|>": 101254,
|
| 968 |
+
"<|LOC_958|>": 101255,
|
| 969 |
+
"<|LOC_959|>": 101256,
|
| 970 |
+
"<|LOC_95|>": 100392,
|
| 971 |
+
"<|LOC_960|>": 101257,
|
| 972 |
+
"<|LOC_961|>": 101258,
|
| 973 |
+
"<|LOC_962|>": 101259,
|
| 974 |
+
"<|LOC_963|>": 101260,
|
| 975 |
+
"<|LOC_964|>": 101261,
|
| 976 |
+
"<|LOC_965|>": 101262,
|
| 977 |
+
"<|LOC_966|>": 101263,
|
| 978 |
+
"<|LOC_967|>": 101264,
|
| 979 |
+
"<|LOC_968|>": 101265,
|
| 980 |
+
"<|LOC_969|>": 101266,
|
| 981 |
+
"<|LOC_96|>": 100393,
|
| 982 |
+
"<|LOC_970|>": 101267,
|
| 983 |
+
"<|LOC_971|>": 101268,
|
| 984 |
+
"<|LOC_972|>": 101269,
|
| 985 |
+
"<|LOC_973|>": 101270,
|
| 986 |
+
"<|LOC_974|>": 101271,
|
| 987 |
+
"<|LOC_975|>": 101272,
|
| 988 |
+
"<|LOC_976|>": 101273,
|
| 989 |
+
"<|LOC_977|>": 101274,
|
| 990 |
+
"<|LOC_978|>": 101275,
|
| 991 |
+
"<|LOC_979|>": 101276,
|
| 992 |
+
"<|LOC_97|>": 100394,
|
| 993 |
+
"<|LOC_980|>": 101277,
|
| 994 |
+
"<|LOC_981|>": 101278,
|
| 995 |
+
"<|LOC_982|>": 101279,
|
| 996 |
+
"<|LOC_983|>": 101280,
|
| 997 |
+
"<|LOC_984|>": 101281,
|
| 998 |
+
"<|LOC_985|>": 101282,
|
| 999 |
+
"<|LOC_986|>": 101283,
|
| 1000 |
+
"<|LOC_987|>": 101284,
|
| 1001 |
+
"<|LOC_988|>": 101285,
|
| 1002 |
+
"<|LOC_989|>": 101286,
|
| 1003 |
+
"<|LOC_98|>": 100395,
|
| 1004 |
+
"<|LOC_990|>": 101287,
|
| 1005 |
+
"<|LOC_991|>": 101288,
|
| 1006 |
+
"<|LOC_992|>": 101289,
|
| 1007 |
+
"<|LOC_993|>": 101290,
|
| 1008 |
+
"<|LOC_994|>": 101291,
|
| 1009 |
+
"<|LOC_995|>": 101292,
|
| 1010 |
+
"<|LOC_996|>": 101293,
|
| 1011 |
+
"<|LOC_997|>": 101294,
|
| 1012 |
+
"<|LOC_998|>": 101295,
|
| 1013 |
+
"<|LOC_999|>": 101296,
|
| 1014 |
+
"<|LOC_99|>": 100396,
|
| 1015 |
+
"<|LOC_9|>": 100306,
|
| 1016 |
+
"<|LOC_BEGIN|>": 101298,
|
| 1017 |
+
"<|LOC_END|>": 101299,
|
| 1018 |
+
"<|LOC_SEP|>": 101300,
|
| 1019 |
+
"<|image_pad|>": 101304,
|
| 1020 |
+
"<|video_pad|>": 101307
|
| 1021 |
+
}
|
chat_template.jinja
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- if not add_generation_prompt is defined -%}
|
| 2 |
+
{%- set add_generation_prompt = true -%}
|
| 3 |
+
{%- endif -%}
|
| 4 |
+
{%- if not cls_token is defined -%}
|
| 5 |
+
{%- set cls_token = "<|begin_of_sentence|>" -%}
|
| 6 |
+
{%- endif -%}
|
| 7 |
+
{%- if not eos_token is defined -%}
|
| 8 |
+
{%- set eos_token = "</s>" -%}
|
| 9 |
+
{%- endif -%}
|
| 10 |
+
{{- cls_token -}}
|
| 11 |
+
{%- for message in messages -%}
|
| 12 |
+
{%- if message["role"] == "user" -%}
|
| 13 |
+
{{- "User: " -}}
|
| 14 |
+
{%- for content in message["content"] -%}
|
| 15 |
+
{%- if content["type"] == "image" -%}
|
| 16 |
+
{{ "<|IMAGE_START|><|IMAGE_PLACEHOLDER|><|IMAGE_END|>" }}
|
| 17 |
+
{%- endif -%}
|
| 18 |
+
{%- endfor -%}
|
| 19 |
+
{%- for content in message["content"] -%}
|
| 20 |
+
{%- if content["type"] == "text" -%}
|
| 21 |
+
{{ content["text"] }}
|
| 22 |
+
{%- endif -%}
|
| 23 |
+
{%- endfor -%}
|
| 24 |
+
{{ "\n" -}}
|
| 25 |
+
{%- elif message["role"] == "assistant" -%}
|
| 26 |
+
{{- "Assistant:\n" -}}
|
| 27 |
+
{%- for content in message["content"] -%}
|
| 28 |
+
{%- if content["type"] == "text" -%}
|
| 29 |
+
{{ content["text"] }}
|
| 30 |
+
{%- endif -%}
|
| 31 |
+
{%- endfor -%}
|
| 32 |
+
{{ eos_token -}}
|
| 33 |
+
{%- elif message["role"] == "system" -%}
|
| 34 |
+
{%- for content in message["content"] -%}
|
| 35 |
+
{%- if content["type"] == "text" -%}
|
| 36 |
+
{{ content["text"] + "\n" }}
|
| 37 |
+
{%- endif -%}
|
| 38 |
+
{%- endfor -%}
|
| 39 |
+
{%- endif -%}
|
| 40 |
+
{%- endfor -%}
|
| 41 |
+
{%- if add_generation_prompt -%}
|
| 42 |
+
{{- "Assistant:\n" -}}
|
| 43 |
+
{%- endif -%}
|
config.json
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"PaddleOCRVLForConditionalGeneration"
|
| 4 |
+
],
|
| 5 |
+
"attention_probs_dropout_prob": 0.0,
|
| 6 |
+
"auto_map": {
|
| 7 |
+
"AutoConfig": "configuration_paddleocr_vl.PaddleOCRVLConfig",
|
| 8 |
+
"AutoModel": "modeling_paddleocr_vl.PaddleOCRVLForConditionalGeneration",
|
| 9 |
+
"AutoModelForCausalLM": "modeling_paddleocr_vl.PaddleOCRVLForConditionalGeneration"
|
| 10 |
+
},
|
| 11 |
+
"compression_ratio": 1.0,
|
| 12 |
+
"head_dim": 128,
|
| 13 |
+
"hidden_act": "silu",
|
| 14 |
+
"hidden_dropout_prob": 0.0,
|
| 15 |
+
"hidden_size": 1024,
|
| 16 |
+
"ignored_index": -100,
|
| 17 |
+
"image_token_id": 100295,
|
| 18 |
+
"intermediate_size": 3072,
|
| 19 |
+
"max_position_embeddings": 131072,
|
| 20 |
+
"max_sequence_length": null,
|
| 21 |
+
"model_type": "paddleocr_vl",
|
| 22 |
+
"num_attention_heads": 16,
|
| 23 |
+
"num_hidden_layers": 18,
|
| 24 |
+
"num_key_value_heads": 2,
|
| 25 |
+
"pad_token_id": 0,
|
| 26 |
+
"rms_norm_eps": 1e-05,
|
| 27 |
+
"rope_scaling": {
|
| 28 |
+
"mrope_section": [
|
| 29 |
+
16,
|
| 30 |
+
24,
|
| 31 |
+
24
|
| 32 |
+
],
|
| 33 |
+
"rope_type": "default",
|
| 34 |
+
"type": "default"
|
| 35 |
+
},
|
| 36 |
+
"rope_theta": 500000,
|
| 37 |
+
"sliding_window": null,
|
| 38 |
+
"tie_word_embeddings": false,
|
| 39 |
+
"torch_dtype": "bfloat16",
|
| 40 |
+
"transformers_version": "4.55.0",
|
| 41 |
+
"use_bias": false,
|
| 42 |
+
"use_cache": false,
|
| 43 |
+
"use_flash_attention": false,
|
| 44 |
+
"video_token_id": 101307,
|
| 45 |
+
"vision_config": {
|
| 46 |
+
"architectures": [
|
| 47 |
+
"PaddleOCRVisionModel"
|
| 48 |
+
],
|
| 49 |
+
"attention_dropout": 0.0,
|
| 50 |
+
"auto_map": {
|
| 51 |
+
"AutoConfig": "configuration_paddleocr_vl.PaddleOCRVLConfig",
|
| 52 |
+
"AutoModel": "modeling_paddleocr_vl.PaddleOCRVisionModel"
|
| 53 |
+
},
|
| 54 |
+
"hidden_act": "gelu_pytorch_tanh",
|
| 55 |
+
"hidden_size": 1152,
|
| 56 |
+
"image_size": 384,
|
| 57 |
+
"intermediate_size": 4304,
|
| 58 |
+
"layer_norm_eps": 1e-06,
|
| 59 |
+
"model_type": "paddleocr_vl",
|
| 60 |
+
"num_attention_heads": 16,
|
| 61 |
+
"num_channels": 3,
|
| 62 |
+
"num_hidden_layers": 27,
|
| 63 |
+
"pad_token_id": 0,
|
| 64 |
+
"patch_size": 14,
|
| 65 |
+
"spatial_merge_size": 2,
|
| 66 |
+
"temporal_patch_size": 2,
|
| 67 |
+
"tokens_per_second": 2,
|
| 68 |
+
"torch_dtype": "bfloat16"
|
| 69 |
+
},
|
| 70 |
+
"vision_start_token_id": 101305,
|
| 71 |
+
"vision_end_token_id": 101306,
|
| 72 |
+
"vocab_size": 103424,
|
| 73 |
+
"weight_share_add_bias": true,
|
| 74 |
+
"use_3d_rope": true,
|
| 75 |
+
"rope_is_neox_style": true
|
| 76 |
+
}
|
configuration_paddleocr_vl.py
ADDED
|
@@ -0,0 +1,191 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
|
| 15 |
+
from transformers.configuration_utils import PretrainedConfig
|
| 16 |
+
from transformers.modeling_rope_utils import rope_config_validation
|
| 17 |
+
|
| 18 |
+
class PaddleOCRVisionConfig(PretrainedConfig):
|
| 19 |
+
model_type = "paddleocr_vl"
|
| 20 |
+
base_config_key = "vision_config"
|
| 21 |
+
|
| 22 |
+
def __init__(
|
| 23 |
+
self,
|
| 24 |
+
hidden_size=768,
|
| 25 |
+
intermediate_size=3072,
|
| 26 |
+
num_hidden_layers=12,
|
| 27 |
+
num_attention_heads=12,
|
| 28 |
+
num_channels=3,
|
| 29 |
+
image_size=224,
|
| 30 |
+
patch_size=14,
|
| 31 |
+
hidden_act="gelu_pytorch_tanh",
|
| 32 |
+
layer_norm_eps=1e-6,
|
| 33 |
+
attention_dropout=0.0,
|
| 34 |
+
spatial_merge_size=2,
|
| 35 |
+
temporal_patch_size=2,
|
| 36 |
+
tokens_per_second=2,
|
| 37 |
+
**kwargs,
|
| 38 |
+
):
|
| 39 |
+
super().__init__(**kwargs)
|
| 40 |
+
|
| 41 |
+
self.hidden_size = hidden_size
|
| 42 |
+
self.intermediate_size = intermediate_size
|
| 43 |
+
self.num_hidden_layers = num_hidden_layers
|
| 44 |
+
self.num_attention_heads = num_attention_heads
|
| 45 |
+
self.num_channels = num_channels
|
| 46 |
+
self.patch_size = patch_size
|
| 47 |
+
self.image_size = image_size
|
| 48 |
+
self.attention_dropout = attention_dropout
|
| 49 |
+
self.layer_norm_eps = layer_norm_eps
|
| 50 |
+
self.hidden_act = hidden_act
|
| 51 |
+
self.spatial_merge_size = spatial_merge_size
|
| 52 |
+
self.temporal_patch_size = temporal_patch_size
|
| 53 |
+
self.tokens_per_second = tokens_per_second
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
class PaddleOCRVLConfig(PretrainedConfig):
|
| 58 |
+
"""
|
| 59 |
+
Configuration class.
|
| 60 |
+
|
| 61 |
+
This class stores the configuration of an Ernie model, defining the model architecture.
|
| 62 |
+
It inherits from PretrainedConfig and can be used to control model outputs.
|
| 63 |
+
"""
|
| 64 |
+
|
| 65 |
+
model_type = "paddleocr_vl"
|
| 66 |
+
keys_to_ignore_at_inference = ["past_key_values"]
|
| 67 |
+
sub_configs = {"vision_config": PaddleOCRVisionConfig}
|
| 68 |
+
|
| 69 |
+
# Default tensor parallel plan for base model `Qwen3`
|
| 70 |
+
base_model_tp_plan = {
|
| 71 |
+
"layers.*.self_attn.q_proj": "colwise",
|
| 72 |
+
"layers.*.self_attn.k_proj": "colwise",
|
| 73 |
+
"layers.*.self_attn.v_proj": "colwise",
|
| 74 |
+
"layers.*.self_attn.o_proj": "rowwise",
|
| 75 |
+
"layers.*.mlp.gate_proj": "colwise",
|
| 76 |
+
"layers.*.mlp.up_proj": "colwise",
|
| 77 |
+
"layers.*.mlp.down_proj": "rowwise",
|
| 78 |
+
}
|
| 79 |
+
base_model_pp_plan = {
|
| 80 |
+
"embed_tokens": (["input_ids"], ["inputs_embeds"]),
|
| 81 |
+
"layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
|
| 82 |
+
"norm": (["hidden_states"], ["hidden_states"]),
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
def __init__(
|
| 86 |
+
self,
|
| 87 |
+
vocab_size=32000,
|
| 88 |
+
hidden_size=768,
|
| 89 |
+
intermediate_size=11008,
|
| 90 |
+
max_position_embeddings=32768,
|
| 91 |
+
num_hidden_layers=2,
|
| 92 |
+
num_attention_heads=2,
|
| 93 |
+
image_token_id=101304,
|
| 94 |
+
video_token_id=101305,
|
| 95 |
+
vision_start_token_id=101306,
|
| 96 |
+
rms_norm_eps=1e-6,
|
| 97 |
+
use_cache=False,
|
| 98 |
+
use_flash_attention=False,
|
| 99 |
+
pad_token_id=0,
|
| 100 |
+
bos_token_id=1,
|
| 101 |
+
eos_token_id=2,
|
| 102 |
+
head_dim=128,
|
| 103 |
+
hidden_act="silu",
|
| 104 |
+
use_bias=False,
|
| 105 |
+
rope_theta=10000,
|
| 106 |
+
weight_share_add_bias=True,
|
| 107 |
+
ignored_index=-100,
|
| 108 |
+
attention_probs_dropout_prob=0.0,
|
| 109 |
+
hidden_dropout_prob=0.0,
|
| 110 |
+
compression_ratio: float = 1.0,
|
| 111 |
+
num_key_value_heads=None,
|
| 112 |
+
max_sequence_length=None,
|
| 113 |
+
tie_word_embeddings=False,
|
| 114 |
+
vision_config=None,
|
| 115 |
+
rope_scaling=None,
|
| 116 |
+
**kwargs,
|
| 117 |
+
):
|
| 118 |
+
"""
|
| 119 |
+
Initialize configuration with default or specified parameters.
|
| 120 |
+
|
| 121 |
+
Args:
|
| 122 |
+
vocab_size (int): Size of the vocabulary (number of unique tokens)
|
| 123 |
+
hidden_size (int): Dimensionality of the encoder layers and the pooler layer
|
| 124 |
+
intermediate_size (int): Dimensionality of the "intermediate" (feed-forward) layer
|
| 125 |
+
max_position_embeddings (int): Maximum sequence length the model can handle
|
| 126 |
+
num_hidden_layers (int): Number of hidden layers in the Transformer encoder
|
| 127 |
+
num_attention_heads (int): Number of attention heads for each attention layer
|
| 128 |
+
rms_norm_eps (float): The epsilon used by the RMS normalization layers
|
| 129 |
+
use_cache (bool): Whether to use caching for faster generation (decoding)
|
| 130 |
+
use_flash_attention (bool): Whether to use FlashAttention for optimized attention computation
|
| 131 |
+
pad_token_id (int): Token ID used for padding sequences
|
| 132 |
+
bos_token_id (int): Token ID used for beginning-of-sequence
|
| 133 |
+
eos_token_id (int): Token ID used for end-of-sequence
|
| 134 |
+
use_bias (bool): Whether to use bias terms in linear layers
|
| 135 |
+
rope_theta (float): The base period of the RoPE embeddings
|
| 136 |
+
weight_share_add_bias (bool): Whether to share bias weights in certain layers
|
| 137 |
+
ignored_index (int): Target value that is ignored during loss computation
|
| 138 |
+
attention_probs_dropout_prob (float): Dropout probability for attention weights
|
| 139 |
+
hidden_dropout_prob (float): Dropout probability for hidden layers
|
| 140 |
+
compression_ratio (float): Ratio for KV cache compression (1.0 = no compression)
|
| 141 |
+
num_key_value_heads (int): Number of key/value heads (for Grouped Query Attention)
|
| 142 |
+
max_sequence_length (int): Maximum sequence length for positional embeddings
|
| 143 |
+
**kwargs: Additional keyword arguments passed to parent class
|
| 144 |
+
"""
|
| 145 |
+
|
| 146 |
+
# Set default for tied embeddings if not specified.
|
| 147 |
+
super().__init__(
|
| 148 |
+
pad_token_id=pad_token_id,
|
| 149 |
+
bos_token_id=bos_token_id,
|
| 150 |
+
eos_token_id=eos_token_id,
|
| 151 |
+
**kwargs,
|
| 152 |
+
)
|
| 153 |
+
if isinstance(vision_config, dict):
|
| 154 |
+
self.vision_config = self.sub_configs["vision_config"](**vision_config)
|
| 155 |
+
elif vision_config is None:
|
| 156 |
+
self.vision_config = self.sub_configs["vision_config"]()
|
| 157 |
+
self.vocab_size = vocab_size
|
| 158 |
+
self.hidden_size = hidden_size
|
| 159 |
+
self.intermediate_size = intermediate_size
|
| 160 |
+
self.max_position_embeddings = max_position_embeddings
|
| 161 |
+
self.num_hidden_layers = num_hidden_layers
|
| 162 |
+
self.num_attention_heads = num_attention_heads
|
| 163 |
+
self.rms_norm_eps = rms_norm_eps
|
| 164 |
+
self.use_cache = use_cache
|
| 165 |
+
self.use_flash_attention = use_flash_attention
|
| 166 |
+
self.pad_token_id = pad_token_id
|
| 167 |
+
self.bos_token_id = bos_token_id
|
| 168 |
+
self.eos_token_id = eos_token_id
|
| 169 |
+
self.image_token_id = image_token_id
|
| 170 |
+
self.video_token_id = video_token_id
|
| 171 |
+
self.vision_start_token_id = vision_start_token_id
|
| 172 |
+
self.head_dim = head_dim
|
| 173 |
+
self.hidden_act=hidden_act
|
| 174 |
+
self.sliding_window = None
|
| 175 |
+
self.hidden_size = hidden_size
|
| 176 |
+
self.use_bias = use_bias
|
| 177 |
+
self.weight_share_add_bias = weight_share_add_bias
|
| 178 |
+
self.rope_theta = rope_theta
|
| 179 |
+
self.ignored_index = ignored_index
|
| 180 |
+
self.attention_probs_dropout_prob = attention_probs_dropout_prob
|
| 181 |
+
self.hidden_dropout_prob = hidden_dropout_prob
|
| 182 |
+
self.compression_ratio = compression_ratio
|
| 183 |
+
self.num_key_value_heads = num_key_value_heads
|
| 184 |
+
self.max_sequence_length = max_sequence_length
|
| 185 |
+
self.rope_scaling = rope_scaling
|
| 186 |
+
if self.rope_scaling is not None and "type" in self.rope_scaling:
|
| 187 |
+
if self.rope_scaling["type"] == "mrope":
|
| 188 |
+
self.rope_scaling["type"] = "default"
|
| 189 |
+
self.rope_scaling["rope_type"] = self.rope_scaling["type"]
|
| 190 |
+
rope_config_validation(self, ignore_keys={"mrope_section"})
|
| 191 |
+
super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
|
docs/CODEX_GPT.md
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Codex 与 GPT 配置
|
| 2 |
+
|
| 3 |
+
本节仅提供脱敏配置、接手提示和可选择执行的 API 示例。**HF token 只能用于 Hugging Face,不能当 OpenAI API key。** 不复制原机的 `auth.json`、原始会话、`.env` 或任何真实 token。
|
| 4 |
+
|
| 5 |
+
## Codex CLI
|
| 6 |
+
|
| 7 |
+
现场 CLI 版本为 `codex-cli 0.167.1`。需要还原版本可安装该版本;想跟随当前版本可去掉版本号。官方也采用 npm 安装方式:[官方 Codex CLI 示例](https://developers.openai.com/cookbook/examples/codex/build_iterative_repair_loops_with_codex)。
|
| 8 |
+
|
| 9 |
+
```bash
|
| 10 |
+
npm install -g @openai/codex@0.167.1
|
| 11 |
+
codex --version
|
| 12 |
+
codex login
|
| 13 |
+
cd /workspace
|
| 14 |
+
codex
|
| 15 |
+
```
|
| 16 |
+
|
| 17 |
+
`codex login` 在新机器重新登录;如果环境没有浏览器,先看 `codex login --help` 使用当前客户端支持的登录方式。原机 Vast 的非交互 shell 若找不到 Node,需要先 `. /opt/nvm/nvm.sh`。
|
| 18 |
+
|
| 19 |
+
当前原机用户配置的非敏感字段如下,模板见 `examples/codex.config.toml`:
|
| 20 |
+
|
| 21 |
+
```toml
|
| 22 |
+
model = "gpt-6.1-sol"
|
| 23 |
+
model_reasoning_effort = "medium"
|
| 24 |
+
approval_policy = "never"
|
| 25 |
+
sandbox_mode = "danger-full-access"
|
| 26 |
+
```
|
| 27 |
+
|
| 28 |
+
这反映当前实例允许全权限执行的配置。新机应按自己的权限要求设置;没有必要为了表格训练照搬全权限。模型名与 reasoning effort 是否可用取决于账号和客户端,不能从本实例可用推断你的 API 账号有同名访问权。配置字段参考 [官方配置文档](https://learn.chatgpt.com/docs/config-file/config-reference)。
|
| 29 |
+
|
| 30 |
+
如果使用自有兼容供应商,在**用户级** `~/.codex/config.toml` 设置 provider;当前官方协议字段仅支持 Responses。只支持 Chat Completions 的 URL 不能直接当成支持 Codex Responses 的入口:[官方 provider 配置](https://learn.chatgpt.com/docs/config-file/config-reference)。
|
| 31 |
+
|
| 32 |
+
```toml
|
| 33 |
+
model = "YOUR_AVAILABLE_MODEL"
|
| 34 |
+
model_provider = "my_provider"
|
| 35 |
+
model_reasoning_effort = "low"
|
| 36 |
+
|
| 37 |
+
[model_providers.my_provider]
|
| 38 |
+
name = "My provider"
|
| 39 |
+
base_url = "https://YOUR_RESPONSES_ENDPOINT/v1"
|
| 40 |
+
env_key = "CUSTOM_OPENAI_API_KEY"
|
| 41 |
+
wire_api = "responses"
|
| 42 |
+
```
|
| 43 |
+
|
| 44 |
+
没有为本次 harness 编造可迁移的私有网关;需要供应商实际提供兼容端点。此仓库没有保存 ChatGPT 账户登录权限。
|
| 45 |
+
|
| 46 |
+
## GPT Python API
|
| 47 |
+
|
| 48 |
+
建议独立于训练环境安装 SDK。通过环境变量提供 key、模型和可选 base URL,示例使用 Responses API 的 `output_text`:[官方快速开始](https://developers.openai.com/api/docs/quickstart)、[官方 Python SDK](https://developers.openai.com/api/reference/python)。
|
| 49 |
+
|
| 50 |
+
```bash
|
| 51 |
+
python -m pip install openai
|
| 52 |
+
read -rsp 'OpenAI API key: ' OPENAI_API_KEY
|
| 53 |
+
export OPENAI_API_KEY
|
| 54 |
+
export OPENAI_MODEL='YOUR_AVAILABLE_MODEL'
|
| 55 |
+
# 可选:export OPENAI_BASE_URL='https://YOUR_RESPONSES_ENDPOINT/v1'
|
| 56 |
+
python /workspace/paddle-sft800-repro/examples/gpt_example.py
|
| 57 |
+
```
|
| 58 |
+
|
| 59 |
+
示例只在你主动执行时发送一次文本请求;本次打包没有调用 GPT API。需要图像伪标签时应另外实现图片输入、JSON结构校验、重试和审查记录,不要只凭能输出HTML便认定标签正确。之前要求的 `gpt-6-sol`、low、8 workers 是历史候选合成方案,后来已改用其他模型;它不是本轮RL训练依赖,也不意味着这里已经重启合成。
|
| 60 |
+
|
| 61 |
+
## HF 登录与下载
|
| 62 |
+
|
| 63 |
+
公开仓库下载不必使用写入 token。需要再次推送时在新机器单独运行 `hf auth login`,把 token 存入本机凭据库,不写进 shell 历史或 Markdown。下载并发可用 `--max-workers 16`;本次大文件上传使用16个workers。
|
| 64 |
+
|
| 65 |
+
## 给新 Codex 的接手提示
|
| 66 |
+
|
| 67 |
+
```text
|
| 68 |
+
先读 /workspace/paddle-sft800-repro/docs/PROJECT_CONTEXT.md、REWARD_AND_VERL.md、
|
| 69 |
+
EVALUATION.md 和 ENVIRONMENT.md,再检查本机 supervisor、GPU、磁盘和运行日志。
|
| 70 |
+
当前 student/RL 初始化是 SFT800,后续只做 DAPO 与类 DAPO。
|
| 71 |
+
两者使用同一份2499条冻结数据和adaptive_v4 reward;不要擅自重做标签或换GT。
|
| 72 |
+
先跑CPU smoke;8卡GPU任务每次只跑一个。推理完成后,CPU评分可与下个模型推理重叠。
|
| 73 |
+
全量需要crop+fullpage,保留HTML/OTSL,并按来源分别报告TEDS和TEDS-S。
|
| 74 |
+
保存每10步与最后一步HF模型;不要删除唯一完整恢复状态。
|
| 75 |
+
完成后报告实际验证结果、输出路径、尚未完成项,不把计划当已完成。
|
| 76 |
+
```
|
| 77 |
+
|
| 78 |
+
提示是项目上下文,不能代替目标机器自己的 AGENTS.md 与实际权限设置。
|
docs/ENVIRONMENT.md
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 新机器迁移与安装
|
| 2 |
+
|
| 3 |
+
## 原机实际环境
|
| 4 |
+
|
| 5 |
+
Ubuntu 24.04、Python 3.12.15、8 × RTX 6000D(每卡约 83.6 GiB)、compute capability12.0、驱动615.71.10;Torch wheel 为 CUDA13.2。完整安装记录见 `environment/main-installed.json`,布局独立环境见 `layout-installed.json`。这些是现场版本,不是声称任意机器运行 `pip install latest` 都会得到同样环境。
|
| 6 |
+
|
| 7 |
+
| 包 | 现场版本 |
|
| 8 |
+
|---|---|
|
| 9 |
+
| torch / torchvision | 2.14.0+cu132 / 0.29.0+cu132 |
|
| 10 |
+
| vLLM | 0.33.0 |
|
| 11 |
+
| VeRL | 0.9.1,另有本包保存的实际源码快照 |
|
| 12 |
+
| Transformers | 5.10.4 |
|
| 13 |
+
| Ray | 2.60.0 |
|
| 14 |
+
| torchdata / tensordict | 0.11.0 / 0.10.0 |
|
| 15 |
+
| pyarrow / datasets | 26.0.1 / 4.8.4 |
|
| 16 |
+
| numpy / scipy / lxml / apted | 2.3.5 / 1.18.1 / 6.1.3 / 1.0.3 |
|
| 17 |
+
| CPU 布局环境 paddlepaddle / paddlex / paddleocr | 3.3.1 / 3.7.2 / 3.7.0 |
|
| 18 |
+
|
| 19 |
+
**已知版本声明冲突**:vLLM0.33.0 声明 Transformers≥5.16.1,而 VeRL0.9.1 声明 Transformers<5.11。现场使用 Transformers5.10.4 加本地适配,并已运行训练;不能用普通全局依赖求解直接重装,否则会升级/冲突。`runtime-pinned.txt` 记录当前运行依赖闭包,采用 `--no-deps` 按实装版本恢复;这不是未经修改的官方兼容矩阵。运行 `pip check` 可能仍报告该冲突,需结合 CPU smoke 和新机器 GPU smoke 判断,不要为了消除提示自动升级 Transformers。
|
| 20 |
+
|
| 21 |
+
## 1. 下载与挂接路径
|
| 22 |
+
|
| 23 |
+
```bash
|
| 24 |
+
python -m pip install huggingface_hub
|
| 25 |
+
hf download CheneyWong/paddleocr-vl16-sft800-rl-repro \
|
| 26 |
+
--local-dir /workspace/paddle-sft800-repro --max-workers 16
|
| 27 |
+
python /workspace/paddle-sft800-repro/scripts/verify_bundle.py
|
| 28 |
+
python /workspace/paddle-sft800-repro/scripts/setup_workspace.py
|
| 29 |
+
```
|
| 30 |
+
|
| 31 |
+
setup 创建实际的 train、eval、datasets 目录;代码和配置复制,大型只读 Parquet 尽量用硬链接,因此布局缓存中的规范绝对路径保持一致。PP-DocLayoutV3 和 SFT800 权重使用目录链接。若目标已有其他实验目录则退出,不会覆盖。当前发布默认支持 `/workspace` 路径,换根目录需显式修改 YAML、Python 常量和 manifest 绑定。
|
| 32 |
+
|
| 33 |
+
数据约16.7GB,重建短/长 crop 分片再占约6.6GB,环境和模型运行缓存另计。训练 checkpoint 保存需要旧恢复状态、新恢复状态与 HF 导出短时共存,建议安装完仍预留至少50GiB磁盘;实际保存代码会按已有 checkpoint 大小复核空间。
|
| 34 |
+
|
| 35 |
+
## 2. 新建主环境
|
| 36 |
+
|
| 37 |
+
先确认新机器的 GPU 架构与驱动支持所选 Torch CUDA wheel。不要在容器里安装或替换宿主 NVIDIA 驱动。下列命令恢复原 CUDA13.2 wheel,不能保证旧驱动或不同架构可直接使用。
|
| 38 |
+
|
| 39 |
+
```bash
|
| 40 |
+
python -m pip install uv
|
| 41 |
+
uv venv --python 3.12 /venv/main
|
| 42 |
+
uv pip install --python /venv/main/bin/python --no-deps \
|
| 43 |
+
--extra-index-url https://download.pytorch.org/whl/cu132 \
|
| 44 |
+
--extra-index-url https://download.pytorch.org/whl/cpu \
|
| 45 |
+
-r /workspace/paddle-sft800-repro/environment/runtime-pinned.txt
|
| 46 |
+
uv pip install --python /venv/main/bin/python --no-deps -e /workspace/train
|
| 47 |
+
/venv/main/bin/python /workspace/paddle-sft800-repro/scripts/install_verl_snapshot.py
|
| 48 |
+
/venv/main/bin/python /workspace/paddle-sft800-repro/scripts/smoke_cpu.py
|
| 49 |
+
```
|
| 50 |
+
|
| 51 |
+
不要在已有任务的 `/venv/main` 原地执行安装。某个精确版本无法从安装源取得时,应先解决 wheel 来源或明确记录替代版本,再在新机验证;本包不含所有第三方二进制 wheels,未声称已在另一台机器完成全新环境安装。`main-installed.txt` 是全环境审计记录,包含许多本任务不需要的包;实际安装用 `runtime-pinned.txt`。
|
| 52 |
+
|
| 53 |
+
`install_verl_snapshot.py` 在当前 Python site-packages 写入一个 `.pth`,优先使用仓库内 `repro/vendor/verl`,避免丢失现场 VeRL 的修订;本地 Paddle 适配通过 `vllm.general_plugins` 注册。ray worker 也必须能看到这个环境。
|
| 54 |
+
|
| 55 |
+
## 3. 布局环境
|
| 56 |
+
|
| 57 |
+
固定 fullpage 缓存已覆盖2,864页;需要重建或处理新图片时安装 CPU 布局环境:
|
| 58 |
+
|
| 59 |
+
```bash
|
| 60 |
+
uv venv --python 3.12 /workspace/eval/.layout-venv
|
| 61 |
+
uv pip install --python /workspace/eval/.layout-venv/bin/python \
|
| 62 |
+
paddlepaddle==3.3.1 paddlex==3.7.2 paddleocr==3.7.0 \
|
| 63 |
+
numpy==2.3.5 opencv-contrib-python==4.10.0.84
|
| 64 |
+
```
|
| 65 |
+
|
| 66 |
+
本包保存的 detector 在 `assets/PP-DocLayoutV3`。setup 将它接到当前用户的 `~/.paddlex/official_models/PP-DocLayoutV3`;完整布局环境记录用于比对,不把整个 venv 跨机复制。
|
| 67 |
+
|
| 68 |
+
## 4. 原 SFT 重训与当前 RL
|
| 69 |
+
|
| 70 |
+
当前 RL 使用仓库根目录 SFT800;原 SFT 从 Base 开始,另行下载:
|
| 71 |
+
|
| 72 |
+
```bash
|
| 73 |
+
hf download PaddlePaddle/PaddleOCR-VL-1.6 --local-dir /workspace/models/PaddleOCR-VL-1.6
|
| 74 |
+
export PYTHONPATH=/workspace/train:/workspace/eval
|
| 75 |
+
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
| 76 |
+
export VERL_PADDLE_PATCH=1 VLLM_PLUGINS=paddle_verl
|
| 77 |
+
/venv/main/bin/torchrun --standalone --nnodes=1 --nproc-per-node=8 \
|
| 78 |
+
-m verl.trainer.sft_trainer --config-path /workspace/train/configs --config-name sft_formula_train
|
| 79 |
+
```
|
| 80 |
+
|
| 81 |
+
原 SFT checkpoint 导出工具是 `/workspace/train/export_verl.py`,先看 `--help` 再按实际 checkpoint 路径导出。当前 RL 使用 `scripts/train_rl.py`;其配置、reward 和保存策略见 [训练说明](REWARD_AND_VERL.md)。`base_rl_specialist_sft.yaml` / `base_rl_sft.py` 是历史专项 teacher 参考,依赖旧1728池和审计 manifest;这些旧 teacher 数据未在本包中另行复制,不能当作直接可跑入口。
|
| 82 |
+
|
| 83 |
+
## 长任务
|
| 84 |
+
|
| 85 |
+
原机通过 supervisor 管理。新机器可按 `examples/rl-training.supervisor.conf` 修改绝对下载目录再注册服务,日志中可查看 step、reward、耗时;不要在已有训练机器再次启动同一输出目录。
|
| 86 |
+
|
| 87 |
+
8 卡原机训练不因本次打包而迁移或停止。本包没有配置自动向外部服务发送数据;OpenAI/HF 登录需在新机器单独完成。
|
docs/EVALUATION.md
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 固定口径的 crop + fullpage 评测
|
| 2 |
+
|
| 3 |
+
本包带原工程 `eval.py`、延后评分入口 `eval_deferred.py`、三段推理 `run_partitioned_eval.py`、CPU 评分 `score_deferred_dpo.py`、公式 v3 归一化、布局缓存和覆盖规则。名称中的173k对应 **173,141条crop**;fullpage为 **2,864页**,不是173k页。
|
| 4 |
+
|
| 5 |
+
## 数据、格式与来源
|
| 6 |
+
|
| 7 |
+
原 Parquet 包含图片 bytes,换机不依赖图片在原机的绝对路径。保留 source、sample_id、image_sha256、HTML/OTSL与结构标记。修订过的 fullpage GT、备用 GT、布局多表顺序保存在 Parquet 和随附 JSON 中;不用重新从较早的数据发布版本下载替代。
|
| 8 |
+
|
| 9 |
+
fullpage pipeline:PP-DocLayoutV3 布局 → 按区域执行 OCR / table / formula prompt → 抽取 table 输出(HTML或OTSL) → 转 HTML 后与 GT 表列表匹配 → TEDS / TEDS-S。正文不是评分表格。OmniDocBench / real5 使用其表格匹配口径;其他来源沿用脚本既有口径,不应混为一套随意匹配。
|
| 10 |
+
|
| 11 |
+
固定布局缓存中包含已人工复核的边界和多表阅读顺序修正,部分根据已知 GT 表数量检查。这是当前受控比较协议,不应称为完全盲测的原始布局性能。新增改动必须同时应用于所有模型。
|
| 12 |
+
|
| 13 |
+
GT或pred出现公式/方言时,两边使用 `formula_cdm/canonicalize_math_v3.py`。当前默认是文本与结构归一化后 edit-distance / TEDS,**不渲染 CDM**。如 `$<-0.5\\mathrm { V}$`、括号定界和普通 `< - 0.5 V` 有对应测试;上下标不能无条件剥除,以免把缺失指数误判正确。TEDS-S 按原始结构计算,与内容指标分开。
|
| 14 |
+
|
| 15 |
+
## 当前三段推理配置
|
| 16 |
+
|
| 17 |
+
| 阶段 | 数量 | 每GPU实例 | 每实例 infer/page batch | max output tokens | context | repetition penalty |
|
| 18 |
+
|---|---:|---:|---:|---:|---:|---:|
|
| 19 |
+
| crop,GT tokens <3500 | 173066 | 4(共32) | 512 / 512 | 4096 | 16384 | 1.0 |
|
| 20 |
+
| crop,GT tokens ≥3500 | 75 | 1(共8) | 512 / 512 | 16384 | 32768 | 1.2 |
|
| 21 |
+
| fullpage pipeline | 2864页 | 4(共32) | 512 / 512 | **8192** | 16384 | 1.0 |
|
| 22 |
+
|
| 23 |
+
batch 是提交上限,不意味着75个长crop能填满所有引擎512并发。短crop沿32个Parquet shards分配,以视觉token总量负载均衡;长crop独立存储,8卡按行分片。最终恢复原始idx,核对图片、GT、source与完整覆盖,不能直接拼分片索引。
|
| 24 |
+
|
| 25 |
+
首次CPU构建分片,会多写约6.6GB;它是派生文件,故不在HF重复上传:
|
| 26 |
+
|
| 27 |
+
```bash
|
| 28 |
+
/venv/main/bin/python /workspace/eval/partition_crop_by_tokens.py --workers 12
|
| 29 |
+
```
|
| 30 |
+
|
| 31 |
+
完整推理(会占用8GPU):
|
| 32 |
+
|
| 33 |
+
```bash
|
| 34 |
+
/venv/main/bin/python /workspace/eval/run_partitioned_eval.py \
|
| 35 |
+
--model-path /workspace/train/runs/formal/sft-formula-v3/exported-checkpoints/step-0800 \
|
| 36 |
+
--out-dir /workspace/eval/runs/repro-sft800
|
| 37 |
+
```
|
| 38 |
+
|
| 39 |
+
该脚本 `--dry-run` 仍会确保派生分片存在;想只看计划且不建大文件,运行 `scripts/eval_plan.py`。
|
| 40 |
+
|
| 41 |
+
## CPU 评分与流水线
|
| 42 |
+
|
| 43 |
+
推理全部完成并生成 `inference-complete.json` 后:
|
| 44 |
+
|
| 45 |
+
```bash
|
| 46 |
+
/venv/main/bin/python /workspace/paddle-sft800-repro/scripts/score_run.py \
|
| 47 |
+
--inference-run /workspace/eval/runs/repro-sft800 \
|
| 48 |
+
--out-dir /workspace/eval/reports/repro-sft800 --label SFT800 --workers 192
|
| 49 |
+
```
|
| 50 |
+
|
| 51 |
+
封装会创建一个空的规范化分数缓存,并显式指定只评分此模型,避免原历史脚本默认找原机的 Base/DPO 旧目录。每50条分一个CPU任务,输出每样本 scores、按来源 summary、同一 MD 内分开的 TEDS 与 TEDS-S 表及 SVG 来源对比图。重复运行使用相同冻结输入,输入变更必须使用新的输出目录。
|
| 52 |
+
|
| 53 |
+
上一模型CPU评分可以与下一模型GPU推理同时运行;两种作业分不同输出目录。两轮GPU推理仍顺序执行。评分 workers 可按CPU/RAM提高到256,留意每进程树缓存与文本长度;不因workers多就扩大 GPU 作业并发。
|
| 54 |
+
|
| 55 |
+
结构-only样本的TEDS缺失,用各指标有效count作为分母。备用GT分别对TEDS和TEDS-S取合法候选最大值;报告记录所用协议。指标内部为0–1,MD展示0–100,差值是百分点。
|
| 56 |
+
|
| 57 |
+
## Teacher 专项评测
|
| 58 |
+
|
| 59 |
+
原始dataset保留7,299条;`quality-exclusions.json` + index按既有规则得到实际评测队列,不能拿7,299当本轮有效样本数。过滤后的基准为2,273条来源记录:TableVerse361、scanned570、camera1342,跨来源可能有同一图片。参考 Base / Student-step800 / Teacher-v4-30 汇总在 `quality-filtered-v2/`。
|
| 60 |
+
|
| 61 |
+
```bash
|
| 62 |
+
/venv/main/bin/python /workspace/eval/teacher-benchmark-v5/evaluate_teacher.py \
|
| 63 |
+
--model /workspace/train/runs/formal/sft-formula-v3/exported-checkpoints/step-0800 \
|
| 64 |
+
--out /workspace/eval/teacher-benchmark-v5/repro-sft800 --workers 128
|
| 65 |
+
```
|
| 66 |
+
|
| 67 |
+
专项入口沿用其既定8192输出上限和共享评分代码;不混同上面的三段全量协议。本包提供既有参考结果,不声称已在新机器重新跑完所有模型。
|
docs/PROJECT_CONTEXT.md
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 前情提要与接手顺序
|
| 2 |
+
|
| 3 |
+
## 要解决的问题
|
| 4 |
+
|
| 5 |
+
以 PaddleOCR-VL-1.6 为基础,改善表格内容和结构识别,主要关注扫描、拍摄、手写、长表、多行多列,以及没有明显框线或混合框线的表格。弱框线中的纵向文字有时应拆成多行/单元格,有时只是一个单元格内的换行,不能仅凭换行机械拆分;要结合全图、对齐、语义与跨行跨列。
|
| 6 |
+
|
| 7 |
+
项目还观察到 SFT800 在一些简单的 MMSci / SciTSR 表格上出现结构退化或复读,因此后续训练数据不仅包含困难表,也补入普通科学表、长宽表和无 span 表,避免只训练复杂样本。
|
| 8 |
+
|
| 9 |
+
## 已经走过的路线
|
| 10 |
+
|
| 11 |
+
1. **原 SFT**:16,091 条,PaddleOCR-VL-1.6 初始化,full parameter,vision / projector / language 均训练,2 epochs 共 1,004 updates。已有比较选定 **step800** 为后续 RL 起点。step800 位于第二个 epoch 内,不是完成两轮的最后一步。
|
| 12 |
+
2. 曾尝试 GRPO、GSPO、DAPO、类 DAPO,并对多个 checkpoint 做 crop + fullpage 比较;后期存在退化,不能预设最后一步最优。后续用户明确仅继续 DAPO 与类 DAPO。旧 DAPO26 / 类 DAPO26 是保留的历史对照,不在本包权重中。
|
| 13 |
+
3. 专项 teacher:从 Base 对旧 RL 池 1,728 条做 full SFT,2 epochs / 108 updates,目标作为未来 OPD 的一个 teacher。专项评测对象是过滤后的 TableVerse、8k-scanned、8k-camera-captured,而不是拿 173k 的数量去定义 teacher 评测。没有证据说明它已在所有来源优于 SFT800。
|
| 14 |
+
4. **当前一轮**:清理伪标签、公式与表格范围问题,扩充普通表和相似 bad cases;共用冻结训练集 **2,499 张**。两种算法都从 SFT800 独立初始化,使用 adaptive_v4 reward,group16,4096 输出上限,repetition_penalty=1.1,8 卡独占顺序训练。每种计划 38 个更新;每 10 步及最后一步保留 HF 模型。实时状态不写死在本文,原机另有 `pipeline-status.json`。
|
| 15 |
+
5. **下一步**:比较每种方法所有保留 checkpoint,再按相对 SFT800 退化、低于 Base、TEDS-S ≤0.9 和复读做 case study;最终再决定 OPD student,以及 Base / SFT800 / 专项 teacher 的组合。本文不把尚未完成的 OPD 称为已训练。
|
| 16 |
+
|
| 17 |
+
## 本轮数据构成
|
| 18 |
+
|
| 19 |
+
| 来源 | 张数 |
|
| 20 |
+
|---|---:|
|
| 21 |
+
| real-scene(原来源) | 316 |
|
| 22 |
+
| real-scene-Qwen-reverified-table-only | 634 |
|
| 23 |
+
| WTW | 661 |
|
| 24 |
+
| M6Doc | 78 |
|
| 25 |
+
| Historical | 4 |
|
| 26 |
+
| MMSci | 185 |
|
| 27 |
+
| SciTSR | 119 |
|
| 28 |
+
| PubTables-1M | 186 |
|
| 29 |
+
| PubTables-v2 | 117 |
|
| 30 |
+
| PubTabNet | 92 |
|
| 31 |
+
| FinTabNet_OTSL | 50 |
|
| 32 |
+
| SynthTabNet | 36 |
|
| 33 |
+
| synth_2d | 21 |
|
| 34 |
+
| 合计 | 2499 |
|
| 35 |
+
|
| 36 |
+
拍摄 1,317、扫描 376、synthetic 57、unknown 749。unknown 不能等同电子文档;手写没有完整独立标签,不能从场景字段推算其准确数量。
|
| 37 |
+
|
| 38 |
+
有 span 1,305、无 span 1,194;rowspan 502、colspan 1,238、两者都有 435。≥30 行的 301 张,≥12 列的 270 张,最长 77 行 / 最宽 27 列;GT 输出 token 最大 3,319。来源和结构统计见 [完整数据分布](../repro/workspace/train/adaptive-rl-v2/DATA-DISTRIBUTION.md)。
|
| 39 |
+
|
| 40 |
+
候选扩量包含图像相似度和结构特征相似度,并非全部由图像检索产生;训练来源保留 `retrieval_origin`。去重与 heldout 隔离依据图片、原始文档身份和既有近重复审查。标签仍有机器审核成分,应保存 review provenance,不要称作全部人工正确。
|
| 41 |
+
|
| 42 |
+
离线候选 3,002 组,排除 503 组无 reward 方差后得到 2,499 张。类 DAPO 离线额外规则会留 2,338 组,但实际训练故意仍用同一份 2,499 张池,差异放在在线 rollout 过滤,便于比较。
|
| 43 |
+
|
| 44 |
+
## 数据与评测约定
|
| 45 |
+
|
| 46 |
+
- 模糊到无法判断的样本排除;马赛克、涂改或不可辨认文字不臆造。可辨认手写需结合全图识别。
|
| 47 |
+
- 不把标题、说明正文、页眉页脚、印章内容混进表格 HTML。多表或正文混入先找已有 boxes;没有 boxes 时才用 PP-DocLayoutV3 裁剪。
|
| 48 |
+
- 不完整商品局部表、背面透印及明确点名排除的图片不进入训练/评测;**完整商品营养标签可以保留**,这是用户后来的修正。
|
| 49 |
+
- 指定 structure-only 图片仅计算 TEDS-S,不进入训练;不可把空缺 TEDS 记成 0。
|
| 50 |
+
- 训练目标识别 HTML / OTSL 后统一到 Paddle 使用的 OTSL。评测保留 raw prediction、HTML 和 OTSL;公式同时对 pred / GT 归一化。
|
| 51 |
+
- 有些 bad case 原 GT 本身跨行跨列或单元格划分错误。已审核的裁剪、备用 GT 和多表阅读顺序修订必须同时作用于 Base / SFT / RL,不能只改单个模型的分数。
|
| 52 |
+
- fullpage 先布局与识别,再抽取表格与 GT 表格匹配评分,不把整页正文当成一个表格比较。
|
| 53 |
+
|
| 54 |
+
## 新机器接手
|
| 55 |
+
|
| 56 |
+
先校验文件、装环境、跑 CPU smoke;在确认 8 卡空闲后启动单个 DAPO 或类 DAPO 任务。生成当前短/长 crop 分片一次即可多模型复用。一个 checkpoint 推理完成后,可让 CPU 评分与下一个 checkpoint 的 GPU 推理并行;同一轮 8 卡不要叠加两个 GPU 作业。
|
| 57 |
+
|
| 58 |
+
此前一次保存因磁盘不足中断,已验证并恢复。包内有保存前空间检查、预存 dataloader 状态、保留全部 HF 导出且仅清理同轮旧恢复文件的实现。恢复与导出顺序很重要,不要直接删除正在写入或唯一可恢复的 checkpoint。
|
docs/REWARD_AND_VERL.md
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Reward、组筛选与 VeRL 配置
|
| 2 |
+
|
| 3 |
+
代码为准:`table_reward_adaptive.py`、`rl_group_filter.py`、`recipe/dapo/dapo_ray_trainer.py`,配置为 `adaptive_v4_dapo.yaml` / `adaptive_v4_minority_dapo.yaml`,两者继承 `rl_common.yaml`。
|
| 4 |
+
|
| 5 |
+
## adaptive_v4 reward
|
| 6 |
+
|
| 7 |
+
```text
|
| 8 |
+
reward = 0.40 × TEDS + 0.60 × TEDS-S
|
| 9 |
+
- 0.10 × (1 - span_score)
|
| 10 |
+
- 0.10 × hallucinated_merge_rate
|
| 11 |
+
- 0.20 × GT_aware_repetition
|
| 12 |
+
- 0.05 × illegal_otsl
|
| 13 |
+
- 0.05 × (1 - tag_balance_score)
|
| 14 |
+
```
|
| 15 |
+
|
| 16 |
+
结构 fatal 错误将 reward 上限设为 0。关闭旧 soft-overlong 长度扣分;达到 4096 上限本身不重复扣分,但因此缺失的内容、错误结构及实际复读仍影响指标。
|
| 17 |
+
|
| 18 |
+
内容的 LaTeX / Unicode / HTML 方言在 pred、GT 两边用固定 v3 规则归一化后计算 TEDS。TEDS-S 按原结构口径计算,不能让内容归一化改变表格结构或分母。此路径默认**不启动 CDM 渲染**。语义等价仅依赖已经实现的规则,不声称任意 LaTeX 都等价可判。完全匹配的结构快捷路径保留同一评分语义。
|
| 19 |
+
|
| 20 |
+
span 不是只比数量:先比起始单元格与合并方向。简单表要求精确范围,span F1 取三次方;复杂表允许相同起点/方向的范围 IoU 部分得分。简单表条件同时满足:span≤4、网格≤200、行≤20、列≤10、最大 span 面积≤16。复杂度只从 GT 计算,预测不能通过虚构 span 获得“困难表”容错。GT 无合并但预测有合并会扣分;两边都无合并没有额外奖励。
|
| 21 |
+
|
| 22 |
+
复读指标参照 GT,考虑重复行、尾部周期及过量重复;GT 本身有重复零、空格或数值不应被误罚。原始 HTML 用栈检查开闭与父子关系;`<table><td></table></td>` 不能因解析器自动修复而获得高分。OTSL 则检查其网格/行结束规则,不套 HTML 闭合标签要求。
|
| 23 |
+
|
| 24 |
+
## DAPO 与类 DAPO
|
| 25 |
+
|
| 26 |
+
两者 reward 与初始权重相同,使用 GRPO 类组内相对优势估计;算法名字为 DAPO 不意味着 `adv_estimator` 字符串必须写 `dapo`。
|
| 27 |
+
|
| 28 |
+
普通 DAPO 排除最终 reward 方差为 0 的组并补采样。类 DAPO 额外排除 **1–2 个 middle + 其余全部 good**,或 **1–2 个 middle + 其余全部 bad**。group16,k=2。
|
| 29 |
+
|
| 30 |
+
| 16 个回答的组成 | 类 DAPO |
|
| 31 |
+
|---|---|
|
| 32 |
+
| 1–2 good,其余 bad;或反过来 | 保留(须有 reward 方差) |
|
| 33 |
+
| 1–2 middle,其余全 good / 全 bad | 排除 |
|
| 34 |
+
| good / middle / bad 三种混合 | 有方差则保留 |
|
| 35 |
+
| 全 middle,但 reward 不同 | 保留 |
|
| 36 |
+
| 任意类别组成,但所有 reward 相同 | 排除并补采样 |
|
| 37 |
+
| reward 非有限值 | 不可用,整组排除 |
|
| 38 |
+
|
| 39 |
+
good 要同时满足 TEDS≥0.90、TEDS-S≥0.95、span≥0.90、闭合≥0.98、复读≤0.01 且无错误。bad 在以下任一项出现时成立:TEDS≤0.70、TEDS-S≤0.70、span≤0.50、闭合<0.80、复读≥0.10 或非法格式 / fatal / parse error。其余有效结果为 middle。公式告警、必要字段缺失等按 invalid 处理,不能伪装成 middle;具体优先级见 `quality()`。
|
| 40 |
+
|
| 41 |
+
过滤发生在 advantage 计算之前,被过滤组不参与更新;不应说其实际训练 advantage 等于 0。保留组内优势来自相对 reward 的中心化/标准化,绝对低分回答仍可能相对组内均值有正优势。补采样上限 `max_num_gen_batches=32`,并非无限重试。
|
| 42 |
+
|
| 43 |
+
## 当前参数
|
| 44 |
+
|
| 45 |
+
| 项目 | 值 |
|
| 46 |
+
|---|---|
|
| 47 |
+
| 初始模型 | SFT800,两种方法独立初始化 |
|
| 48 |
+
| 模式 | full parameter;FSDP2;vision / projector / language 均训练 |
|
| 49 |
+
| GPU | 8 卡独占;每卡一个训练 rollout vLLM 引擎;TP=1 |
|
| 50 |
+
| 学习率 | 2e-7 |
|
| 51 |
+
| 训练 prompt batch | 全局 64 |
|
| 52 |
+
| 每批补采样候选 prompt | 全局 128 |
|
| 53 |
+
| group size | 16 |
|
| 54 |
+
| Actor / log-prob micro batch | 每卡 4 |
|
| 55 |
+
| sampling | temperature=0.7、top_p=0.95、top_k=-1、repetition_penalty=1.1 |
|
| 56 |
+
| 长度 | prompt cap4096、response cap4096、vLLM context16384 |
|
| 57 |
+
| vLLM 并发 | max_num_seqs512、max_num_batched_tokens16384 |
|
| 58 |
+
| vLLM 显存 | gpu_memory_utilization0.3;显式 KV cache 8 GiB / engine |
|
| 59 |
+
| eager / cache | eager;无 prefix cache;mm_processor_cache_gb0;free_cache_engine |
|
| 60 |
+
| reward | 192 个共享 CPU workers,轮内动态取队列 |
|
| 61 |
+
| Actor | gradient checkpointing;param/optimizer offload;不用 torch.compile |
|
| 62 |
+
| loss | token-mean;clip low0.20 / high0.28;无 KL loss / KL reward |
|
| 63 |
+
| 计划更新数 | 38 = 2 × floor(2499 / 128) |
|
| 64 |
+
| 保存 | step10、20、30、38 的 HF 导出;运行中保留最新完整恢复状态 |
|
| 65 |
+
|
| 66 |
+
`total_epochs=38` 是为了过滤补采样预留的遍历上限,`total_training_steps=38` 才是实际停止条件,不能解释为训练 38 个 epochs。由于过滤、补采样、截取最终训练 batch,实际样本暴露次数不严格等于每张两次。
|
| 67 |
+
|
| 68 |
+
`rollout.data_parallel_size=1` 是单个 vLLM replica 内参数;8 卡训练会建立多个 rollout replicas,不能把它误解成只用一张卡。候选筛选和独立评测可用每卡多个 vLLM;本轮 VeRL 训练不是每卡四���引擎。
|
| 69 |
+
|
| 70 |
+
192 workers 是本机资源配置,不是其他机器的硬要求。迁移脚本默认读取原配置,不自动按 CPU 核数放大;较小机器需一并调整 Ray num_cpus、object_store_memory、reward workers / budget 文件。不要只改 YAML 中一个字段。
|
| 71 |
+
|
| 72 |
+
## 启动
|
| 73 |
+
|
| 74 |
+
完成环境安装和 CPU smoke 后:
|
| 75 |
+
|
| 76 |
+
```bash
|
| 77 |
+
/venv/main/bin/python /workspace/paddle-sft800-repro/scripts/train_rl.py --algorithm dapo
|
| 78 |
+
/venv/main/bin/python /workspace/paddle-sft800-repro/scripts/train_rl.py --algorithm minority_dapo
|
| 79 |
+
# 或依次执行两者:--algorithm both
|
| 80 |
+
```
|
| 81 |
+
|
| 82 |
+
脚本调用现有训练生命周期,独占 8 GPU 锁,自动恢复本机同算法完整 checkpoint,并归档 HF 导出。它只使用已冻结的训练数据,不重新调用 Qwen 筛选。为长任务配套 supervisor 模板见 `examples/rl-training.supervisor.conf`。
|
| 83 |
+
|
| 84 |
+
原 SFT 的参数:full、2epochs、global32、micro1/GPU、lr2e-6 cosine、warmup0.03、max_length12288、SDPA、FSDP2、fp32 model / bf16 compute。重新训练需另行下载 Base;本包根目录的 SFT800 不能冒充 Base 初始化。
|
environment/layout-installed.json
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"python": "3.12.15",
|
| 3 |
+
"packages": {
|
| 4 |
+
"soupsieve": "4.1",
|
| 5 |
+
"py-cpuinfo": "9.0.0",
|
| 6 |
+
"more-itertools": "11.2.1",
|
| 7 |
+
"aiohappyeyeballs": "2.7.1",
|
| 8 |
+
"typing-inspection": "0.4.5",
|
| 9 |
+
"PyYAML": "6.0.2",
|
| 10 |
+
"et_xmlfile": "2.0.0",
|
| 11 |
+
"aiosignal": "1.4.0",
|
| 12 |
+
"threadpoolctl": "3.8.0",
|
| 13 |
+
"ujson": "6.1.0",
|
| 14 |
+
"annotated-types": "0.8.0",
|
| 15 |
+
"six": "1.17.0",
|
| 16 |
+
"hf-xet": "1.8.0",
|
| 17 |
+
"MarkupSafe": "3.0.4",
|
| 18 |
+
"encutils": "1.0.0",
|
| 19 |
+
"jiter": "0.19.0",
|
| 20 |
+
"imagesize": "2.0.1",
|
| 21 |
+
"frozenlist": "1.8.0",
|
| 22 |
+
"psutil": "7.2.2",
|
| 23 |
+
"cssselect": "1.6.0",
|
| 24 |
+
"cloudpickle": "3.1.2",
|
| 25 |
+
"pyclipper": "1.4.0",
|
| 26 |
+
"python-bidi": "0.6.12",
|
| 27 |
+
"pycparser": "3.12",
|
| 28 |
+
"idna": "3.20",
|
| 29 |
+
"yarl": "1.25.1",
|
| 30 |
+
"typing_extensions": "4.16.0",
|
| 31 |
+
"ftfy": "6.3.1",
|
| 32 |
+
"prettytable": "3.18.0",
|
| 33 |
+
"sniffio": "1.3.1",
|
| 34 |
+
"regex": "2026.11.9",
|
| 35 |
+
"certifi": "2026.10.23",
|
| 36 |
+
"colorlog": "6.12.0",
|
| 37 |
+
"truststore": "0.10.5",
|
| 38 |
+
"premailer": "3.10.0",
|
| 39 |
+
"pydantic_core": "2.50.4",
|
| 40 |
+
"propcache": "0.5.4",
|
| 41 |
+
"multidict": "7.2.1",
|
| 42 |
+
"cachetools": "8.0.0",
|
| 43 |
+
"tiktoken": "0.14.0",
|
| 44 |
+
"latex2mathml": "3.81.1",
|
| 45 |
+
"charset-normalizer": "3.5.2",
|
| 46 |
+
"h11": "0.16.0",
|
| 47 |
+
"crc32c": "2.9.post0",
|
| 48 |
+
"safetensors": "0.8.0",
|
| 49 |
+
"sentencepiece": "0.2.3",
|
| 50 |
+
"click": "8.5.1",
|
| 51 |
+
"beautifulsoup4": "4.15.0",
|
| 52 |
+
"einops": "0.8.2",
|
| 53 |
+
"cffi": "2.1.1",
|
| 54 |
+
"requests": "2.34.2",
|
| 55 |
+
"packaging": "26.3",
|
| 56 |
+
"opt-einsum": "3.3.0",
|
| 57 |
+
"filelock": "4.2.10",
|
| 58 |
+
"httpx2": "2.16.0",
|
| 59 |
+
"pypdfium2": "5.15.2",
|
| 60 |
+
"Jinja2": "3.1.6",
|
| 61 |
+
"httpx": "0.28.1",
|
| 62 |
+
"aistudio_sdk": "0.3.9",
|
| 63 |
+
"python-dateutil": "4.0.0",
|
| 64 |
+
"tokenizers": "0.23.3",
|
| 65 |
+
"httpcore": "1.0.9",
|
| 66 |
+
"attrs": "26.1.0",
|
| 67 |
+
"opencv-contrib-python": "4.10.0.84",
|
| 68 |
+
"tqdm": "4.70.1",
|
| 69 |
+
"ruamel.yaml": "0.19.1",
|
| 70 |
+
"httpcore2": "2.16.0",
|
| 71 |
+
"urllib3": "2.8.0",
|
| 72 |
+
"modelscope-hub": "0.7.1",
|
| 73 |
+
"anyio": "4.16.0",
|
| 74 |
+
"cssutils": "2.15.0",
|
| 75 |
+
"chardet": "7.7.0",
|
| 76 |
+
"fsspec": "2026.9.0",
|
| 77 |
+
"protobuf": "7.36.2",
|
| 78 |
+
"wcwidth": "0.9.4",
|
| 79 |
+
"aiohttp": "3.14.5",
|
| 80 |
+
"paddleocr": "3.7.0",
|
| 81 |
+
"pydantic": "2.14.3",
|
| 82 |
+
"cryptography": "50.0.2",
|
| 83 |
+
"pillow": "13.0.0",
|
| 84 |
+
"joblib": "1.6.0",
|
| 85 |
+
"shapely": "2.2.0",
|
| 86 |
+
"narwhals": "2.28.0",
|
| 87 |
+
"lxml": "6.1.3",
|
| 88 |
+
"huggingface_hub": "2.4.0",
|
| 89 |
+
"openpyxl": "3.1.5",
|
| 90 |
+
"bce-python-sdk": "0.9.79",
|
| 91 |
+
"future": "1.0.0",
|
| 92 |
+
"pycryptodome": "4.4.0",
|
| 93 |
+
"setuptools": "84.2.1",
|
| 94 |
+
"networkx": "3.7",
|
| 95 |
+
"tzdata": "2026.6",
|
| 96 |
+
"numpy": "2.3.5",
|
| 97 |
+
"scikit-learn": "1.9.1",
|
| 98 |
+
"paddlex": "3.7.2",
|
| 99 |
+
"scipy": "1.18.1",
|
| 100 |
+
"pandas": "3.1.1",
|
| 101 |
+
"openai": "3.49.0",
|
| 102 |
+
"modelscope": "1.40.3",
|
| 103 |
+
"paddlepaddle": "3.3.1"
|
| 104 |
+
}
|
| 105 |
+
}
|
environment/layout-installed.txt
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
aiohappyeyeballs==2.7.1
|
| 2 |
+
aiohttp==3.14.5
|
| 3 |
+
aiosignal==1.4.0
|
| 4 |
+
aistudio_sdk==0.3.9
|
| 5 |
+
annotated-types==0.8.0
|
| 6 |
+
anyio==4.16.0
|
| 7 |
+
attrs==26.1.0
|
| 8 |
+
bce-python-sdk==0.9.79
|
| 9 |
+
beautifulsoup4==4.15.0
|
| 10 |
+
cachetools==8.0.0
|
| 11 |
+
certifi==2026.10.23
|
| 12 |
+
cffi==2.1.1
|
| 13 |
+
chardet==7.7.0
|
| 14 |
+
charset-normalizer==3.5.2
|
| 15 |
+
click==8.5.1
|
| 16 |
+
cloudpickle==3.1.2
|
| 17 |
+
colorlog==6.12.0
|
| 18 |
+
crc32c==2.9.post0
|
| 19 |
+
cryptography==50.0.2
|
| 20 |
+
cssselect==1.6.0
|
| 21 |
+
cssutils==2.15.0
|
| 22 |
+
einops==0.8.2
|
| 23 |
+
encutils==1.0.0
|
| 24 |
+
et_xmlfile==2.0.0
|
| 25 |
+
filelock==4.2.10
|
| 26 |
+
frozenlist==1.8.0
|
| 27 |
+
fsspec==2026.9.0
|
| 28 |
+
ftfy==6.3.1
|
| 29 |
+
future==1.0.0
|
| 30 |
+
h11==0.16.0
|
| 31 |
+
hf-xet==1.8.0
|
| 32 |
+
httpcore==1.0.9
|
| 33 |
+
httpcore2==2.16.0
|
| 34 |
+
httpx==0.28.1
|
| 35 |
+
httpx2==2.16.0
|
| 36 |
+
huggingface_hub==2.4.0
|
| 37 |
+
idna==3.20
|
| 38 |
+
imagesize==2.0.1
|
| 39 |
+
Jinja2==3.1.6
|
| 40 |
+
jiter==0.19.0
|
| 41 |
+
joblib==1.6.0
|
| 42 |
+
latex2mathml==3.81.1
|
| 43 |
+
lxml==6.1.3
|
| 44 |
+
MarkupSafe==3.0.4
|
| 45 |
+
modelscope==1.40.3
|
| 46 |
+
modelscope-hub==0.7.1
|
| 47 |
+
more-itertools==11.2.1
|
| 48 |
+
multidict==7.2.1
|
| 49 |
+
narwhals==2.28.0
|
| 50 |
+
networkx==3.7
|
| 51 |
+
numpy==2.3.5
|
| 52 |
+
openai==3.49.0
|
| 53 |
+
opencv-contrib-python==4.10.0.84
|
| 54 |
+
openpyxl==3.1.5
|
| 55 |
+
opt-einsum==3.3.0
|
| 56 |
+
packaging==26.3
|
| 57 |
+
paddleocr==3.7.0
|
| 58 |
+
paddlepaddle==3.3.1
|
| 59 |
+
paddlex==3.7.2
|
| 60 |
+
pandas==3.1.1
|
| 61 |
+
pillow==13.0.0
|
| 62 |
+
premailer==3.10.0
|
| 63 |
+
prettytable==3.18.0
|
| 64 |
+
propcache==0.5.4
|
| 65 |
+
protobuf==7.36.2
|
| 66 |
+
psutil==7.2.2
|
| 67 |
+
py-cpuinfo==9.0.0
|
| 68 |
+
pyclipper==1.4.0
|
| 69 |
+
pycparser==3.12
|
| 70 |
+
pycryptodome==4.4.0
|
| 71 |
+
pydantic==2.14.3
|
| 72 |
+
pydantic_core==2.50.4
|
| 73 |
+
pypdfium2==5.15.2
|
| 74 |
+
python-bidi==0.6.12
|
| 75 |
+
python-dateutil==4.0.0
|
| 76 |
+
PyYAML==6.0.2
|
| 77 |
+
regex==2026.11.9
|
| 78 |
+
requests==2.34.2
|
| 79 |
+
ruamel.yaml==0.19.1
|
| 80 |
+
safetensors==0.8.0
|
| 81 |
+
scikit-learn==1.9.1
|
| 82 |
+
scipy==1.18.1
|
| 83 |
+
sentencepiece==0.2.3
|
| 84 |
+
setuptools==84.2.1
|
| 85 |
+
shapely==2.2.0
|
| 86 |
+
six==1.17.0
|
| 87 |
+
sniffio==1.3.1
|
| 88 |
+
soupsieve==4.1
|
| 89 |
+
threadpoolctl==3.8.0
|
| 90 |
+
tiktoken==0.14.0
|
| 91 |
+
tokenizers==0.23.3
|
| 92 |
+
tqdm==4.70.1
|
| 93 |
+
truststore==0.10.5
|
| 94 |
+
typing-inspection==0.4.5
|
| 95 |
+
typing_extensions==4.16.0
|
| 96 |
+
tzdata==2026.6
|
| 97 |
+
ujson==6.1.0
|
| 98 |
+
urllib3==2.8.0
|
| 99 |
+
wcwidth==0.9.4
|
| 100 |
+
yarl==1.25.1
|
environment/main-installed.json
ADDED
|
@@ -0,0 +1,329 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"python": "3.12.15",
|
| 3 |
+
"packages": {
|
| 4 |
+
"pip": "26.2.2",
|
| 5 |
+
"wheel": "0.48.0",
|
| 6 |
+
"partial-json-parser": "0.2.1.1.post7",
|
| 7 |
+
"mcp-types": "2.3.0",
|
| 8 |
+
"py-cpuinfo": "9.0.0",
|
| 9 |
+
"httptools": "0.9.0",
|
| 10 |
+
"aiosignal": "1.4.0",
|
| 11 |
+
"cloudpickle": "3.1.2",
|
| 12 |
+
"fastapi": "0.136.3",
|
| 13 |
+
"opentelemetry-exporter-otlp": "1.45.2",
|
| 14 |
+
"fastapi-cloud-cli": "0.27.1",
|
| 15 |
+
"oss-harmony": "0.0.12",
|
| 16 |
+
"typer": "0.29.1",
|
| 17 |
+
"tokenspeed-triton": "3.9.10.post20261028",
|
| 18 |
+
"fastapi-cli": "0.0.33",
|
| 19 |
+
"cuda-toolkit": "13.2.1",
|
| 20 |
+
"annotated-doc": "0.0.6",
|
| 21 |
+
"pynvvideocodec": "2.0.4",
|
| 22 |
+
"cuda-python": "13.5.0",
|
| 23 |
+
"python-json-logger": "4.2.0",
|
| 24 |
+
"nvidia-nvjitlink": "13.5.37",
|
| 25 |
+
"agent-detector": "2.0.0",
|
| 26 |
+
"z3-solver": "4.15.4.0",
|
| 27 |
+
"jiter": "0.19.0",
|
| 28 |
+
"llguidance": "1.7.6",
|
| 29 |
+
"docstring_parser": "0.18.0",
|
| 30 |
+
"multidict": "7.2.1",
|
| 31 |
+
"typing-inspection": "0.4.5",
|
| 32 |
+
"python-dotenv": "1.2.4",
|
| 33 |
+
"annotated-types": "0.8.0",
|
| 34 |
+
"referencing": "0.37.0",
|
| 35 |
+
"prometheus_client": "0.26.0",
|
| 36 |
+
"interegular": "0.3.3",
|
| 37 |
+
"cbor2": "6.2.1",
|
| 38 |
+
"nvidia-ml-py": "13.615.71",
|
| 39 |
+
"nvidia-cuda-nvrtc": "13.2.78",
|
| 40 |
+
"sentencepiece": "0.2.3",
|
| 41 |
+
"tabulate": "0.10.0",
|
| 42 |
+
"orjson": "3.13.0",
|
| 43 |
+
"prometheus-fastapi-instrumentator": "8.1.0",
|
| 44 |
+
"nvidia-cutlass-dsl": "4.8.0",
|
| 45 |
+
"cuda-pathfinder": "1.8.6",
|
| 46 |
+
"instanttensor": "0.2.3",
|
| 47 |
+
"setproctitle": "1.3.8",
|
| 48 |
+
"nvidia-cusparselt-cu13": "0.8.1",
|
| 49 |
+
"detect-installer": "0.2.1",
|
| 50 |
+
"opentelemetry-exporter-otlp-proto-grpc": "1.45.2",
|
| 51 |
+
"httpx": "0.28.1",
|
| 52 |
+
"blake3": "1.0.11",
|
| 53 |
+
"fastar": "0.12.0",
|
| 54 |
+
"depyf": "0.20.0",
|
| 55 |
+
"rpds-py": "2026.9.1",
|
| 56 |
+
"rignore": "0.8.1",
|
| 57 |
+
"lark": "1.2.2",
|
| 58 |
+
"ml_dtypes": "0.6.0",
|
| 59 |
+
"sse-starlette": "3.7.0",
|
| 60 |
+
"jsonschema": "4.26.0",
|
| 61 |
+
"regex": "2026.11.9",
|
| 62 |
+
"python-multipart": "0.0.33",
|
| 63 |
+
"nvidia-cufile": "1.17.1.22",
|
| 64 |
+
"pydantic_core": "2.50.4",
|
| 65 |
+
"attrs": "26.1.0",
|
| 66 |
+
"Jinja2": "3.1.6",
|
| 67 |
+
"frozenlist": "1.8.0",
|
| 68 |
+
"pydantic-extra-types": "2.11.1",
|
| 69 |
+
"yarl": "1.25.1",
|
| 70 |
+
"mdurl": "0.1.2",
|
| 71 |
+
"lm-format-enforcer": "0.11.3",
|
| 72 |
+
"opentelemetry-exporter-http-transport": "0.66b2",
|
| 73 |
+
"MarkupSafe": "3.0.4",
|
| 74 |
+
"aiohappyeyeballs": "2.7.1",
|
| 75 |
+
"certifi": "2026.10.23",
|
| 76 |
+
"propcache": "0.5.4",
|
| 77 |
+
"shellingham": "1.5.4",
|
| 78 |
+
"sniffio": "1.3.1",
|
| 79 |
+
"opentelemetry-semantic-conventions-ai": "0.5.1",
|
| 80 |
+
"loguru": "0.7.3",
|
| 81 |
+
"opentelemetry-exporter-otlp-common": "0.66b2",
|
| 82 |
+
"watchfiles": "1.3.0",
|
| 83 |
+
"pycparser": "3.12",
|
| 84 |
+
"urllib3": "2.8.0",
|
| 85 |
+
"outlines_core": "0.2.14",
|
| 86 |
+
"safetensors": "0.8.0",
|
| 87 |
+
"cachetools": "8.0.0",
|
| 88 |
+
"nvidia-cutlass-dsl-libs-cu13": "4.8.0",
|
| 89 |
+
"charset-normalizer": "3.5.2",
|
| 90 |
+
"email-validator": "2.3.0",
|
| 91 |
+
"nvtx": "0.2.15",
|
| 92 |
+
"PyJWT": "2.15.1",
|
| 93 |
+
"starlette": "1.10.0",
|
| 94 |
+
"jmespath": "1.1.0",
|
| 95 |
+
"pybase64": "1.5.1",
|
| 96 |
+
"nvidia-cutlass-dsl-libs-cu12": "4.8.0",
|
| 97 |
+
"fastsafetensors": "0.5.0",
|
| 98 |
+
"astor": "0.8.1",
|
| 99 |
+
"opentelemetry-exporter-otlp-proto-http": "1.45.2",
|
| 100 |
+
"opentelemetry-proto": "1.45.2",
|
| 101 |
+
"opentelemetry-exporter-otlp-proto-common": "1.45.2",
|
| 102 |
+
"tokenspeed-mla": "2.0.3",
|
| 103 |
+
"msgspec": "0.22.0",
|
| 104 |
+
"requests": "2.34.2",
|
| 105 |
+
"nvidia-cuda-nvcc": "13.5.37",
|
| 106 |
+
"dill": "0.4.1",
|
| 107 |
+
"jsonschema-specifications": "2025.9.1",
|
| 108 |
+
"einops": "0.8.2",
|
| 109 |
+
"torch_c_dlpack_ext": "0.1.5",
|
| 110 |
+
"nvidia-cuda-crt": "13.5.37",
|
| 111 |
+
"pydantic-settings": "2.16.2",
|
| 112 |
+
"nvidia-cutlass-dsl-libs-base": "4.8.0",
|
| 113 |
+
"nvidia-cusparse": "12.7.10.1",
|
| 114 |
+
"httpcore": "1.0.9",
|
| 115 |
+
"rich-toolkit": "0.20.6",
|
| 116 |
+
"tiktoken": "0.14.0",
|
| 117 |
+
"nvidia-cufft": "12.2.0.46",
|
| 118 |
+
"websockets": "17.4",
|
| 119 |
+
"nvidia-nvvm": "13.5.37",
|
| 120 |
+
"ninja": "1.13.2",
|
| 121 |
+
"aiohttp": "3.14.5",
|
| 122 |
+
"nccl4py": "0.6.0",
|
| 123 |
+
"nvidia-nvtx": "13.2.75",
|
| 124 |
+
"xgrammar": "0.2.8",
|
| 125 |
+
"uvicorn": "0.54.0",
|
| 126 |
+
"cuda-core": "1.2.2",
|
| 127 |
+
"llvmlite": "0.47.0",
|
| 128 |
+
"ijson": "3.6.0",
|
| 129 |
+
"pydantic": "2.14.3",
|
| 130 |
+
"opentelemetry-api": "1.45.2",
|
| 131 |
+
"mpmath": "1.3.0",
|
| 132 |
+
"protobuf": "7.36.2",
|
| 133 |
+
"torchaudio": "2.11.0+cpu",
|
| 134 |
+
"nvidia-curand": "10.4.2.55",
|
| 135 |
+
"mistral_common": "1.13.1",
|
| 136 |
+
"uvloop": "0.23.0",
|
| 137 |
+
"grpcio": "1.85.1",
|
| 138 |
+
"markdown-it-py": "4.3.0",
|
| 139 |
+
"nccl-extensions": "0.1.0",
|
| 140 |
+
"opentelemetry-sdk": "1.45.2",
|
| 141 |
+
"torchcodec": "0.17.0+cu132",
|
| 142 |
+
"nvidia-cusolver": "12.2.0.1",
|
| 143 |
+
"cuda-tile": "1.8.0",
|
| 144 |
+
"nvidia-cuda-cupti": "13.2.75",
|
| 145 |
+
"cffi": "2.1.1",
|
| 146 |
+
"nvidia-nvshmem-cu13": "3.4.5",
|
| 147 |
+
"model-hosting-container-standards": "0.1.16",
|
| 148 |
+
"pillow": "13.0.0",
|
| 149 |
+
"cuda-bindings": "13.5.0",
|
| 150 |
+
"opencv-python-headless": "5.0.0.93",
|
| 151 |
+
"rich": "15.0.0",
|
| 152 |
+
"mcp": "2.3.0",
|
| 153 |
+
"compressed-tensors": "0.17.0",
|
| 154 |
+
"nvidia-cuda-runtime": "13.2.75",
|
| 155 |
+
"nvidia-nccl-cu13": "2.30.7",
|
| 156 |
+
"cryptography": "50.0.2",
|
| 157 |
+
"supervisor": "4.3.0",
|
| 158 |
+
"humming-kernels": "0.1.19",
|
| 159 |
+
"dnspython": "2.9.2",
|
| 160 |
+
"quack-kernels": "0.6.5",
|
| 161 |
+
"opentelemetry-semantic-conventions": "0.66b2",
|
| 162 |
+
"torchvision": "0.29.0+cu132",
|
| 163 |
+
"nvidia-cudnn-cu13": "9.24.0.43",
|
| 164 |
+
"googleapis-common-protos": "1.76.0",
|
| 165 |
+
"nvidia-cublas": "13.4.0.1",
|
| 166 |
+
"sentry-sdk": "2.74.0",
|
| 167 |
+
"apache-tvm-ffi": "0.1.11",
|
| 168 |
+
"nvidia-cutlass-dsl-libs-core": "4.8.0",
|
| 169 |
+
"setuptools": "80.10.2",
|
| 170 |
+
"networkx": "3.7",
|
| 171 |
+
"triton": "3.8.0",
|
| 172 |
+
"pycountry": "26.2.16",
|
| 173 |
+
"numba": "0.65.0",
|
| 174 |
+
"numpy": "2.3.5",
|
| 175 |
+
"nvidia-cudnn-frontend": "1.32.3",
|
| 176 |
+
"sympy": "1.14.0",
|
| 177 |
+
"openai": "3.49.0",
|
| 178 |
+
"nvidia-cuda-cccl": "13.3.5.0.6",
|
| 179 |
+
"anthropic": "2.8.0",
|
| 180 |
+
"py-spy": "0.4.3",
|
| 181 |
+
"tilelang": "0.1.12",
|
| 182 |
+
"vllm": "0.33.0",
|
| 183 |
+
"torch": "2.14.0+cu132",
|
| 184 |
+
"flashinfer-python": "0.7.2.post1",
|
| 185 |
+
"flashinfer-cubin": "0.7.2.post1",
|
| 186 |
+
"flashinfer-jit-cache-sm90a": "0.7.2.post1+cu130",
|
| 187 |
+
"flashinfer-jit-cache-sm89": "0.7.2.post1+cu130",
|
| 188 |
+
"flashinfer-jit-cache-sm80": "0.7.2.post1+cu130",
|
| 189 |
+
"flashinfer-jit-cache-sm120f": "0.7.2.post1+cu130",
|
| 190 |
+
"flashinfer-jit-cache-sm103a": "0.7.2.post1+cu130",
|
| 191 |
+
"flashinfer-jit-cache-sm100a": "0.7.2.post1+cu130",
|
| 192 |
+
"flashinfer-jit-cache": "0.7.2.post1+cu130",
|
| 193 |
+
"torchdata": "0.11.0",
|
| 194 |
+
"lazy-importer": "0.1.0",
|
| 195 |
+
"tokenizers": "0.22.2",
|
| 196 |
+
"codetiming": "1.4.0",
|
| 197 |
+
"sortedcontainers": "2.4.0",
|
| 198 |
+
"aiohttp-cors": "0.8.1",
|
| 199 |
+
"defusedxml": "0.7.1",
|
| 200 |
+
"qwen-vl-utils": "0.0.14",
|
| 201 |
+
"cycler": "0.12.1",
|
| 202 |
+
"xxhash": "4.1.0",
|
| 203 |
+
"addict": "2.4.0",
|
| 204 |
+
"transformers-stream-generator": "0.0.5",
|
| 205 |
+
"distro": "1.9.0",
|
| 206 |
+
"pyvers": "0.1.0",
|
| 207 |
+
"colorful": "0.5.8",
|
| 208 |
+
"attrdict": "2.0.1",
|
| 209 |
+
"tensorboard-data-server": "0.7.2",
|
| 210 |
+
"opentelemetry-exporter-prometheus": "0.66b2",
|
| 211 |
+
"opencensus-context": "0.1.3",
|
| 212 |
+
"rouge": "1.0.1",
|
| 213 |
+
"kiwisolver": "1.5.1",
|
| 214 |
+
"msgpack": "1.2.4",
|
| 215 |
+
"zipp": "4.1.2",
|
| 216 |
+
"pyparsing": "3.3.3",
|
| 217 |
+
"zstandard": "0.25.0",
|
| 218 |
+
"crcmod": "1.7",
|
| 219 |
+
"dacite": "1.9.2",
|
| 220 |
+
"contourpy": "1.4.0",
|
| 221 |
+
"wrapt": "2.5.2",
|
| 222 |
+
"binpacking": "2.0.1",
|
| 223 |
+
"absl-py": "2.5.1",
|
| 224 |
+
"python-discovery": "1.9.5",
|
| 225 |
+
"importlib_metadata": "10.2.0",
|
| 226 |
+
"pylatexenc": "2.11",
|
| 227 |
+
"packaging": "25.0",
|
| 228 |
+
"omegaconf": "2.3.1",
|
| 229 |
+
"smart_open": "9.0.3",
|
| 230 |
+
"distlib": "0.4.4",
|
| 231 |
+
"json_repair": "0.64.9",
|
| 232 |
+
"oss2": "2.19.1",
|
| 233 |
+
"proto-plus": "1.29.0",
|
| 234 |
+
"hjson": "3.1.0",
|
| 235 |
+
"pyasn1": "0.6.5",
|
| 236 |
+
"Markdown": "3.11.3",
|
| 237 |
+
"simplejson": "4.3.0",
|
| 238 |
+
"cpm-kernels": "1.0.11",
|
| 239 |
+
"tensordict": "0.10.0",
|
| 240 |
+
"modelscope-hub": "0.7.1",
|
| 241 |
+
"Werkzeug": "3.1.9",
|
| 242 |
+
"multiprocess": "0.70.19",
|
| 243 |
+
"fsspec": "2026.2.0",
|
| 244 |
+
"antlr4-python3-runtime": "4.9.3",
|
| 245 |
+
"google-api-core": "2.44.0",
|
| 246 |
+
"pybind11": "3.1.0",
|
| 247 |
+
"google-auth": "2.62.2",
|
| 248 |
+
"accelerate": "1.15.0",
|
| 249 |
+
"aliyun-python-sdk-kms": "2.16.5",
|
| 250 |
+
"opencensus": "0.11.4",
|
| 251 |
+
"virtualenv": "21.14.5",
|
| 252 |
+
"datasets": "4.8.4",
|
| 253 |
+
"pyasn1_modules": "0.4.2",
|
| 254 |
+
"joblib": "1.6.0",
|
| 255 |
+
"cwevals": "0.0.1b1",
|
| 256 |
+
"trl": "0.29.1",
|
| 257 |
+
"hydra-core": "1.3.8",
|
| 258 |
+
"aliyun-python-sdk-core": "2.16.1",
|
| 259 |
+
"huggingface_hub": "1.33.0",
|
| 260 |
+
"tensorboard": "2.21.0",
|
| 261 |
+
"peft": "0.20.0",
|
| 262 |
+
"av": "19.0.2",
|
| 263 |
+
"pycryptodome": "4.4.0",
|
| 264 |
+
"liger_kernel": "0.8.5",
|
| 265 |
+
"fonttools": "4.68.0",
|
| 266 |
+
"nltk": "3.10.4",
|
| 267 |
+
"verl": "0.9.1",
|
| 268 |
+
"ms_swift": "4.6.0",
|
| 269 |
+
"wandb": "0.31.0",
|
| 270 |
+
"matplotlib": "3.11.2",
|
| 271 |
+
"pyarrow": "26.0.1",
|
| 272 |
+
"deepspeed": "0.19.9",
|
| 273 |
+
"scipy": "1.18.1",
|
| 274 |
+
"pandas": "3.1.1",
|
| 275 |
+
"transformers": "5.10.4",
|
| 276 |
+
"ray": "2.60.0",
|
| 277 |
+
"modelscope": "1.40.3",
|
| 278 |
+
"apted": "1.0.3",
|
| 279 |
+
"RapidFuzz": "3.14.6",
|
| 280 |
+
"lxml": "6.1.3",
|
| 281 |
+
"verl-paddle-local-adapter": "0.1.0",
|
| 282 |
+
"python-Levenshtein": "0.27.5",
|
| 283 |
+
"Levenshtein": "0.27.5",
|
| 284 |
+
"soupsieve": "4.1",
|
| 285 |
+
"beautifulsoup4": "4.15.0",
|
| 286 |
+
"anyio": "4.15.1",
|
| 287 |
+
"asttokens": "3.0.2",
|
| 288 |
+
"click": "8.5.0",
|
| 289 |
+
"comm": "0.2.3",
|
| 290 |
+
"debugpy": "1.8.22",
|
| 291 |
+
"executing": "2.3.0",
|
| 292 |
+
"filelock": "4.2.4",
|
| 293 |
+
"h11": "0.16.0",
|
| 294 |
+
"hf-xet": "1.7.0",
|
| 295 |
+
"httpcore2": "2.16.0",
|
| 296 |
+
"httpx2": "2.16.0",
|
| 297 |
+
"idna": "3.20",
|
| 298 |
+
"ipykernel": "7.4.0",
|
| 299 |
+
"ipython": "9.17.1",
|
| 300 |
+
"ipython_pygments_lexers": "1.1.1",
|
| 301 |
+
"ipywidgets": "8.1.9",
|
| 302 |
+
"jedi": "0.20.1",
|
| 303 |
+
"jupyter_client": "8.10.0",
|
| 304 |
+
"jupyter_core": "5.9.1",
|
| 305 |
+
"jupyterlab_widgets": "3.0.17",
|
| 306 |
+
"matplotlib-inline": "0.2.2",
|
| 307 |
+
"nest-asyncio2": "1.7.4",
|
| 308 |
+
"parso": "0.8.7",
|
| 309 |
+
"pexpect": "4.9.0",
|
| 310 |
+
"platformdirs": "4.13.5",
|
| 311 |
+
"prompt_toolkit": "3.0.53",
|
| 312 |
+
"psutil": "7.2.2",
|
| 313 |
+
"ptyprocess": "0.7.0",
|
| 314 |
+
"pure_eval": "0.2.4",
|
| 315 |
+
"Pygments": "2.21.0",
|
| 316 |
+
"python-dateutil": "2.9.0.post0",
|
| 317 |
+
"PyYAML": "6.0.3",
|
| 318 |
+
"pyzmq": "27.2.0",
|
| 319 |
+
"six": "1.17.0",
|
| 320 |
+
"stack-data": "0.6.3",
|
| 321 |
+
"tornado": "6.6",
|
| 322 |
+
"tqdm": "4.70.1",
|
| 323 |
+
"traitlets": "5.16.1",
|
| 324 |
+
"truststore": "0.10.5",
|
| 325 |
+
"typing_extensions": "4.16.0",
|
| 326 |
+
"wcwidth": "0.9.3",
|
| 327 |
+
"widgetsnbextension": "4.0.16"
|
| 328 |
+
}
|
| 329 |
+
}
|
environment/main-installed.txt
ADDED
|
@@ -0,0 +1,324 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
absl-py==2.5.1
|
| 2 |
+
accelerate==1.15.0
|
| 3 |
+
addict==2.4.0
|
| 4 |
+
agent-detector==2.0.0
|
| 5 |
+
aiohappyeyeballs==2.7.1
|
| 6 |
+
aiohttp==3.14.5
|
| 7 |
+
aiohttp-cors==0.8.1
|
| 8 |
+
aiosignal==1.4.0
|
| 9 |
+
aliyun-python-sdk-core==2.16.1
|
| 10 |
+
aliyun-python-sdk-kms==2.16.5
|
| 11 |
+
annotated-doc==0.0.6
|
| 12 |
+
annotated-types==0.8.0
|
| 13 |
+
anthropic==2.8.0
|
| 14 |
+
antlr4-python3-runtime==4.9.3
|
| 15 |
+
anyio==4.15.1
|
| 16 |
+
apache-tvm-ffi==0.1.11
|
| 17 |
+
apted==1.0.3
|
| 18 |
+
astor==0.8.1
|
| 19 |
+
asttokens==3.0.2
|
| 20 |
+
attrdict==2.0.1
|
| 21 |
+
attrs==26.1.0
|
| 22 |
+
av==19.0.2
|
| 23 |
+
beautifulsoup4==4.15.0
|
| 24 |
+
binpacking==2.0.1
|
| 25 |
+
blake3==1.0.11
|
| 26 |
+
cachetools==8.0.0
|
| 27 |
+
cbor2==6.2.1
|
| 28 |
+
certifi==2026.10.23
|
| 29 |
+
cffi==2.1.1
|
| 30 |
+
charset-normalizer==3.5.2
|
| 31 |
+
click==8.5.0
|
| 32 |
+
cloudpickle==3.1.2
|
| 33 |
+
codetiming==1.4.0
|
| 34 |
+
colorful==0.5.8
|
| 35 |
+
comm==0.2.3
|
| 36 |
+
compressed-tensors==0.17.0
|
| 37 |
+
contourpy==1.4.0
|
| 38 |
+
cpm-kernels==1.0.11
|
| 39 |
+
crcmod==1.7
|
| 40 |
+
cryptography==50.0.2
|
| 41 |
+
cuda-bindings==13.5.0
|
| 42 |
+
cuda-core==1.2.2
|
| 43 |
+
cuda-pathfinder==1.8.6
|
| 44 |
+
cuda-python==13.5.0
|
| 45 |
+
cuda-tile==1.8.0
|
| 46 |
+
cuda-toolkit==13.2.1
|
| 47 |
+
cwevals==0.0.1b1
|
| 48 |
+
cycler==0.12.1
|
| 49 |
+
dacite==1.9.2
|
| 50 |
+
datasets==4.8.4
|
| 51 |
+
debugpy==1.8.22
|
| 52 |
+
deepspeed==0.19.9
|
| 53 |
+
defusedxml==0.7.1
|
| 54 |
+
depyf==0.20.0
|
| 55 |
+
detect-installer==0.2.1
|
| 56 |
+
dill==0.4.1
|
| 57 |
+
distlib==0.4.4
|
| 58 |
+
distro==1.9.0
|
| 59 |
+
dnspython==2.9.2
|
| 60 |
+
docstring_parser==0.18.0
|
| 61 |
+
einops==0.8.2
|
| 62 |
+
email-validator==2.3.0
|
| 63 |
+
executing==2.3.0
|
| 64 |
+
fastapi==0.136.3
|
| 65 |
+
fastapi-cli==0.0.33
|
| 66 |
+
fastapi-cloud-cli==0.27.1
|
| 67 |
+
fastar==0.12.0
|
| 68 |
+
fastsafetensors==0.5.0
|
| 69 |
+
filelock==4.2.4
|
| 70 |
+
flashinfer-cubin==0.7.2.post1
|
| 71 |
+
flashinfer-jit-cache==0.7.2.post1+cu130
|
| 72 |
+
flashinfer-jit-cache-sm100a==0.7.2.post1+cu130
|
| 73 |
+
flashinfer-jit-cache-sm103a==0.7.2.post1+cu130
|
| 74 |
+
flashinfer-jit-cache-sm120f==0.7.2.post1+cu130
|
| 75 |
+
flashinfer-jit-cache-sm80==0.7.2.post1+cu130
|
| 76 |
+
flashinfer-jit-cache-sm89==0.7.2.post1+cu130
|
| 77 |
+
flashinfer-jit-cache-sm90a==0.7.2.post1+cu130
|
| 78 |
+
flashinfer-python==0.7.2.post1
|
| 79 |
+
fonttools==4.68.0
|
| 80 |
+
frozenlist==1.8.0
|
| 81 |
+
fsspec==2026.2.0
|
| 82 |
+
google-api-core==2.44.0
|
| 83 |
+
google-auth==2.62.2
|
| 84 |
+
googleapis-common-protos==1.76.0
|
| 85 |
+
grpcio==1.85.1
|
| 86 |
+
h11==0.16.0
|
| 87 |
+
hf-xet==1.7.0
|
| 88 |
+
hjson==3.1.0
|
| 89 |
+
httpcore==1.0.9
|
| 90 |
+
httpcore2==2.16.0
|
| 91 |
+
httptools==0.9.0
|
| 92 |
+
httpx==0.28.1
|
| 93 |
+
httpx2==2.16.0
|
| 94 |
+
huggingface_hub==1.33.0
|
| 95 |
+
humming-kernels==0.1.19
|
| 96 |
+
hydra-core==1.3.8
|
| 97 |
+
idna==3.20
|
| 98 |
+
ijson==3.6.0
|
| 99 |
+
importlib_metadata==10.2.0
|
| 100 |
+
instanttensor==0.2.3
|
| 101 |
+
interegular==0.3.3
|
| 102 |
+
ipykernel==7.4.0
|
| 103 |
+
ipython==9.17.1
|
| 104 |
+
ipython_pygments_lexers==1.1.1
|
| 105 |
+
ipywidgets==8.1.9
|
| 106 |
+
jedi==0.20.1
|
| 107 |
+
Jinja2==3.1.6
|
| 108 |
+
jiter==0.19.0
|
| 109 |
+
jmespath==1.1.0
|
| 110 |
+
joblib==1.6.0
|
| 111 |
+
json_repair==0.64.9
|
| 112 |
+
jsonschema==4.26.0
|
| 113 |
+
jsonschema-specifications==2025.9.1
|
| 114 |
+
jupyter_client==8.10.0
|
| 115 |
+
jupyter_core==5.9.1
|
| 116 |
+
jupyterlab_widgets==3.0.17
|
| 117 |
+
kiwisolver==1.5.1
|
| 118 |
+
lark==1.2.2
|
| 119 |
+
lazy-importer==0.1.0
|
| 120 |
+
Levenshtein==0.27.5
|
| 121 |
+
liger_kernel==0.8.5
|
| 122 |
+
llguidance==1.7.6
|
| 123 |
+
llvmlite==0.47.0
|
| 124 |
+
lm-format-enforcer==0.11.3
|
| 125 |
+
loguru==0.7.3
|
| 126 |
+
lxml==6.1.3
|
| 127 |
+
Markdown==3.11.3
|
| 128 |
+
markdown-it-py==4.3.0
|
| 129 |
+
MarkupSafe==3.0.4
|
| 130 |
+
matplotlib==3.11.2
|
| 131 |
+
matplotlib-inline==0.2.2
|
| 132 |
+
mcp==2.3.0
|
| 133 |
+
mcp-types==2.3.0
|
| 134 |
+
mdurl==0.1.2
|
| 135 |
+
mistral_common==1.13.1
|
| 136 |
+
ml_dtypes==0.6.0
|
| 137 |
+
model-hosting-container-standards==0.1.16
|
| 138 |
+
modelscope==1.40.3
|
| 139 |
+
modelscope-hub==0.7.1
|
| 140 |
+
mpmath==1.3.0
|
| 141 |
+
ms_swift==4.6.0
|
| 142 |
+
msgpack==1.2.4
|
| 143 |
+
msgspec==0.22.0
|
| 144 |
+
multidict==7.2.1
|
| 145 |
+
multiprocess==0.70.19
|
| 146 |
+
nccl-extensions==0.1.0
|
| 147 |
+
nccl4py==0.6.0
|
| 148 |
+
nest-asyncio2==1.7.4
|
| 149 |
+
networkx==3.7
|
| 150 |
+
ninja==1.13.2
|
| 151 |
+
nltk==3.10.4
|
| 152 |
+
numba==0.65.0
|
| 153 |
+
numpy==2.3.5
|
| 154 |
+
nvidia-cublas==13.4.0.1
|
| 155 |
+
nvidia-cuda-cccl==13.3.5.0.6
|
| 156 |
+
nvidia-cuda-crt==13.5.37
|
| 157 |
+
nvidia-cuda-cupti==13.2.75
|
| 158 |
+
nvidia-cuda-nvcc==13.5.37
|
| 159 |
+
nvidia-cuda-nvrtc==13.2.78
|
| 160 |
+
nvidia-cuda-runtime==13.2.75
|
| 161 |
+
nvidia-cudnn-cu13==9.24.0.43
|
| 162 |
+
nvidia-cudnn-frontend==1.32.3
|
| 163 |
+
nvidia-cufft==12.2.0.46
|
| 164 |
+
nvidia-cufile==1.17.1.22
|
| 165 |
+
nvidia-curand==10.4.2.55
|
| 166 |
+
nvidia-cusolver==12.2.0.1
|
| 167 |
+
nvidia-cusparse==12.7.10.1
|
| 168 |
+
nvidia-cusparselt-cu13==0.8.1
|
| 169 |
+
nvidia-cutlass-dsl==4.8.0
|
| 170 |
+
nvidia-cutlass-dsl-libs-base==4.8.0
|
| 171 |
+
nvidia-cutlass-dsl-libs-core==4.8.0
|
| 172 |
+
nvidia-cutlass-dsl-libs-cu12==4.8.0
|
| 173 |
+
nvidia-cutlass-dsl-libs-cu13==4.8.0
|
| 174 |
+
nvidia-ml-py==13.615.71
|
| 175 |
+
nvidia-nccl-cu13==2.30.7
|
| 176 |
+
nvidia-nvjitlink==13.5.37
|
| 177 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 178 |
+
nvidia-nvtx==13.2.75
|
| 179 |
+
nvidia-nvvm==13.5.37
|
| 180 |
+
nvtx==0.2.15
|
| 181 |
+
omegaconf==2.3.1
|
| 182 |
+
openai==3.49.0
|
| 183 |
+
opencensus==0.11.4
|
| 184 |
+
opencensus-context==0.1.3
|
| 185 |
+
opencv-python-headless==5.0.0.93
|
| 186 |
+
opentelemetry-api==1.45.2
|
| 187 |
+
opentelemetry-exporter-http-transport==0.66b2
|
| 188 |
+
opentelemetry-exporter-otlp==1.45.2
|
| 189 |
+
opentelemetry-exporter-otlp-common==0.66b2
|
| 190 |
+
opentelemetry-exporter-otlp-proto-common==1.45.2
|
| 191 |
+
opentelemetry-exporter-otlp-proto-grpc==1.45.2
|
| 192 |
+
opentelemetry-exporter-otlp-proto-http==1.45.2
|
| 193 |
+
opentelemetry-exporter-prometheus==0.66b2
|
| 194 |
+
opentelemetry-proto==1.45.2
|
| 195 |
+
opentelemetry-sdk==1.45.2
|
| 196 |
+
opentelemetry-semantic-conventions==0.66b2
|
| 197 |
+
opentelemetry-semantic-conventions-ai==0.5.1
|
| 198 |
+
orjson==3.13.0
|
| 199 |
+
oss-harmony==0.0.12
|
| 200 |
+
oss2==2.19.1
|
| 201 |
+
outlines_core==0.2.14
|
| 202 |
+
packaging==25.0
|
| 203 |
+
pandas==3.1.1
|
| 204 |
+
parso==0.8.7
|
| 205 |
+
partial-json-parser==0.2.1.1.post7
|
| 206 |
+
peft==0.20.0
|
| 207 |
+
pexpect==4.9.0
|
| 208 |
+
pillow==13.0.0
|
| 209 |
+
pip==26.2.2
|
| 210 |
+
platformdirs==4.13.5
|
| 211 |
+
prometheus-fastapi-instrumentator==8.1.0
|
| 212 |
+
prometheus_client==0.26.0
|
| 213 |
+
prompt_toolkit==3.0.53
|
| 214 |
+
propcache==0.5.4
|
| 215 |
+
proto-plus==1.29.0
|
| 216 |
+
protobuf==7.36.2
|
| 217 |
+
psutil==7.2.2
|
| 218 |
+
ptyprocess==0.7.0
|
| 219 |
+
pure_eval==0.2.4
|
| 220 |
+
py-cpuinfo==9.0.0
|
| 221 |
+
py-spy==0.4.3
|
| 222 |
+
pyarrow==26.0.1
|
| 223 |
+
pyasn1==0.6.5
|
| 224 |
+
pyasn1_modules==0.4.2
|
| 225 |
+
pybase64==1.5.1
|
| 226 |
+
pybind11==3.1.0
|
| 227 |
+
pycountry==26.2.16
|
| 228 |
+
pycparser==3.12
|
| 229 |
+
pycryptodome==4.4.0
|
| 230 |
+
pydantic==2.14.3
|
| 231 |
+
pydantic-extra-types==2.11.1
|
| 232 |
+
pydantic-settings==2.16.2
|
| 233 |
+
pydantic_core==2.50.4
|
| 234 |
+
Pygments==2.21.0
|
| 235 |
+
PyJWT==2.15.1
|
| 236 |
+
pylatexenc==2.11
|
| 237 |
+
pynvvideocodec==2.0.4
|
| 238 |
+
pyparsing==3.3.3
|
| 239 |
+
python-dateutil==2.9.0.post0
|
| 240 |
+
python-discovery==1.9.5
|
| 241 |
+
python-dotenv==1.2.4
|
| 242 |
+
python-json-logger==4.2.0
|
| 243 |
+
python-Levenshtein==0.27.5
|
| 244 |
+
python-multipart==0.0.33
|
| 245 |
+
pyvers==0.1.0
|
| 246 |
+
PyYAML==6.0.3
|
| 247 |
+
pyzmq==27.2.0
|
| 248 |
+
quack-kernels==0.6.5
|
| 249 |
+
qwen-vl-utils==0.0.14
|
| 250 |
+
RapidFuzz==3.14.6
|
| 251 |
+
ray==2.60.0
|
| 252 |
+
referencing==0.37.0
|
| 253 |
+
regex==2026.11.9
|
| 254 |
+
requests==2.34.2
|
| 255 |
+
rich==15.0.0
|
| 256 |
+
rich-toolkit==0.20.6
|
| 257 |
+
rignore==0.8.1
|
| 258 |
+
rouge==1.0.1
|
| 259 |
+
rpds-py==2026.9.1
|
| 260 |
+
safetensors==0.8.0
|
| 261 |
+
scipy==1.18.1
|
| 262 |
+
sentencepiece==0.2.3
|
| 263 |
+
sentry-sdk==2.74.0
|
| 264 |
+
setproctitle==1.3.8
|
| 265 |
+
setuptools==80.10.2
|
| 266 |
+
shellingham==1.5.4
|
| 267 |
+
simplejson==4.3.0
|
| 268 |
+
six==1.17.0
|
| 269 |
+
smart_open==9.0.3
|
| 270 |
+
sniffio==1.3.1
|
| 271 |
+
sortedcontainers==2.4.0
|
| 272 |
+
soupsieve==4.1
|
| 273 |
+
sse-starlette==3.7.0
|
| 274 |
+
stack-data==0.6.3
|
| 275 |
+
starlette==1.10.0
|
| 276 |
+
supervisor==4.3.0
|
| 277 |
+
sympy==1.14.0
|
| 278 |
+
tabulate==0.10.0
|
| 279 |
+
tensorboard==2.21.0
|
| 280 |
+
tensorboard-data-server==0.7.2
|
| 281 |
+
tensordict==0.10.0
|
| 282 |
+
tiktoken==0.14.0
|
| 283 |
+
tilelang==0.1.12
|
| 284 |
+
tokenizers==0.22.2
|
| 285 |
+
tokenspeed-mla==2.0.3
|
| 286 |
+
tokenspeed-triton==3.9.10.post20261028
|
| 287 |
+
torch==2.14.0+cu132
|
| 288 |
+
torch_c_dlpack_ext==0.1.5
|
| 289 |
+
torchaudio==2.11.0+cpu
|
| 290 |
+
torchcodec==0.17.0+cu132
|
| 291 |
+
torchdata==0.11.0
|
| 292 |
+
torchvision==0.29.0+cu132
|
| 293 |
+
tornado==6.6
|
| 294 |
+
tqdm==4.70.1
|
| 295 |
+
traitlets==5.16.1
|
| 296 |
+
transformers==5.10.4
|
| 297 |
+
transformers-stream-generator==0.0.5
|
| 298 |
+
triton==3.8.0
|
| 299 |
+
trl==0.29.1
|
| 300 |
+
truststore==0.10.5
|
| 301 |
+
typer==0.29.1
|
| 302 |
+
typing-inspection==0.4.5
|
| 303 |
+
typing_extensions==4.16.0
|
| 304 |
+
urllib3==2.8.0
|
| 305 |
+
uvicorn==0.54.0
|
| 306 |
+
uvloop==0.23.0
|
| 307 |
+
verl==0.9.1
|
| 308 |
+
verl-paddle-local-adapter==0.1.0
|
| 309 |
+
virtualenv==21.14.5
|
| 310 |
+
vllm==0.33.0
|
| 311 |
+
wandb==0.31.0
|
| 312 |
+
watchfiles==1.3.0
|
| 313 |
+
wcwidth==0.9.3
|
| 314 |
+
websockets==17.4
|
| 315 |
+
Werkzeug==3.1.9
|
| 316 |
+
wheel==0.48.0
|
| 317 |
+
widgetsnbextension==4.0.16
|
| 318 |
+
wrapt==2.5.2
|
| 319 |
+
xgrammar==0.2.8
|
| 320 |
+
xxhash==4.1.0
|
| 321 |
+
yarl==1.25.1
|
| 322 |
+
z3-solver==4.15.4.0
|
| 323 |
+
zipp==4.1.2
|
| 324 |
+
zstandard==0.25.0
|
environment/runtime-pinned.txt
ADDED
|
@@ -0,0 +1,256 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Exact installed runtime dependency closure. Install --no-deps; see ENVIRONMENT.md.
|
| 2 |
+
absl-py==2.5.1
|
| 3 |
+
accelerate==1.15.0
|
| 4 |
+
agent-detector==2.0.0
|
| 5 |
+
aiohappyeyeballs==2.7.1
|
| 6 |
+
aiohttp==3.14.5
|
| 7 |
+
aiohttp-cors==0.8.1
|
| 8 |
+
aiosignal==1.4.0
|
| 9 |
+
annotated-doc==0.0.6
|
| 10 |
+
annotated-types==0.8.0
|
| 11 |
+
anthropic==2.8.0
|
| 12 |
+
antlr4-python3-runtime==4.9.3
|
| 13 |
+
anyio==4.15.1
|
| 14 |
+
apache-tvm-ffi==0.1.11
|
| 15 |
+
apted==1.0.3
|
| 16 |
+
astor==0.8.1
|
| 17 |
+
attrs==26.1.0
|
| 18 |
+
beautifulsoup4==4.15.0
|
| 19 |
+
blake3==1.0.11
|
| 20 |
+
cachetools==8.0.0
|
| 21 |
+
cbor2==6.2.1
|
| 22 |
+
certifi==2026.10.23
|
| 23 |
+
cffi==2.1.1
|
| 24 |
+
charset-normalizer==3.5.2
|
| 25 |
+
click==8.5.0
|
| 26 |
+
cloudpickle==3.1.2
|
| 27 |
+
codetiming==1.4.0
|
| 28 |
+
colorful==0.5.8
|
| 29 |
+
compressed-tensors==0.17.0
|
| 30 |
+
cryptography==50.0.2
|
| 31 |
+
cuda-bindings==13.5.0
|
| 32 |
+
cuda-core==1.2.2
|
| 33 |
+
cuda-pathfinder==1.8.6
|
| 34 |
+
cuda-python==13.5.0
|
| 35 |
+
cuda-tile==1.8.0
|
| 36 |
+
cuda-toolkit==13.2.1
|
| 37 |
+
cwevals==0.0.1b1
|
| 38 |
+
datasets==4.8.4
|
| 39 |
+
depyf==0.20.0
|
| 40 |
+
detect-installer==0.2.1
|
| 41 |
+
dill==0.4.1
|
| 42 |
+
distlib==0.4.4
|
| 43 |
+
distro==1.9.0
|
| 44 |
+
dnspython==2.9.2
|
| 45 |
+
docstring_parser==0.18.0
|
| 46 |
+
einops==0.8.2
|
| 47 |
+
email-validator==2.3.0
|
| 48 |
+
fastapi==0.136.3
|
| 49 |
+
fastapi-cli==0.0.33
|
| 50 |
+
fastapi-cloud-cli==0.27.1
|
| 51 |
+
fastar==0.12.0
|
| 52 |
+
fastsafetensors==0.5.0
|
| 53 |
+
filelock==4.2.4
|
| 54 |
+
flashinfer-python==0.7.2.post1
|
| 55 |
+
frozenlist==1.8.0
|
| 56 |
+
fsspec==2026.2.0
|
| 57 |
+
google-api-core==2.44.0
|
| 58 |
+
google-auth==2.62.2
|
| 59 |
+
googleapis-common-protos==1.76.0
|
| 60 |
+
grpcio==1.85.1
|
| 61 |
+
h11==0.16.0
|
| 62 |
+
hf-xet==1.7.0
|
| 63 |
+
httpcore==1.0.9
|
| 64 |
+
httpcore2==2.16.0
|
| 65 |
+
httptools==0.9.0
|
| 66 |
+
httpx==0.28.1
|
| 67 |
+
httpx2==2.16.0
|
| 68 |
+
huggingface_hub==1.33.0
|
| 69 |
+
humming-kernels==0.1.19
|
| 70 |
+
hydra-core==1.3.8
|
| 71 |
+
idna==3.20
|
| 72 |
+
ijson==3.6.0
|
| 73 |
+
importlib_metadata==10.2.0
|
| 74 |
+
instanttensor==0.2.3
|
| 75 |
+
interegular==0.3.3
|
| 76 |
+
Jinja2==3.1.6
|
| 77 |
+
jiter==0.19.0
|
| 78 |
+
jmespath==1.1.0
|
| 79 |
+
jsonschema==4.26.0
|
| 80 |
+
jsonschema-specifications==2025.9.1
|
| 81 |
+
lark==1.2.2
|
| 82 |
+
lazy-importer==0.1.0
|
| 83 |
+
Levenshtein==0.27.5
|
| 84 |
+
llguidance==1.7.6
|
| 85 |
+
llvmlite==0.47.0
|
| 86 |
+
lm-format-enforcer==0.11.3
|
| 87 |
+
loguru==0.7.3
|
| 88 |
+
lxml==6.1.3
|
| 89 |
+
Markdown==3.11.3
|
| 90 |
+
markdown-it-py==4.3.0
|
| 91 |
+
MarkupSafe==3.0.4
|
| 92 |
+
mcp==2.3.0
|
| 93 |
+
mcp-types==2.3.0
|
| 94 |
+
mdurl==0.1.2
|
| 95 |
+
mistral_common==1.13.1
|
| 96 |
+
ml_dtypes==0.6.0
|
| 97 |
+
model-hosting-container-standards==0.1.16
|
| 98 |
+
mpmath==1.3.0
|
| 99 |
+
msgpack==1.2.4
|
| 100 |
+
msgspec==0.22.0
|
| 101 |
+
multidict==7.2.1
|
| 102 |
+
multiprocess==0.70.19
|
| 103 |
+
nccl-extensions==0.1.0
|
| 104 |
+
nccl4py==0.6.0
|
| 105 |
+
networkx==3.7
|
| 106 |
+
ninja==1.13.2
|
| 107 |
+
numba==0.65.0
|
| 108 |
+
numpy==2.3.5
|
| 109 |
+
nvidia-cublas==13.4.0.1
|
| 110 |
+
nvidia-cuda-cccl==13.3.5.0.6
|
| 111 |
+
nvidia-cuda-crt==13.5.37
|
| 112 |
+
nvidia-cuda-cupti==13.2.75
|
| 113 |
+
nvidia-cuda-nvcc==13.5.37
|
| 114 |
+
nvidia-cuda-nvrtc==13.2.78
|
| 115 |
+
nvidia-cuda-runtime==13.2.75
|
| 116 |
+
nvidia-cudnn-cu13==9.24.0.43
|
| 117 |
+
nvidia-cudnn-frontend==1.32.3
|
| 118 |
+
nvidia-cufft==12.2.0.46
|
| 119 |
+
nvidia-cufile==1.17.1.22
|
| 120 |
+
nvidia-curand==10.4.2.55
|
| 121 |
+
nvidia-cusolver==12.2.0.1
|
| 122 |
+
nvidia-cusparse==12.7.10.1
|
| 123 |
+
nvidia-cusparselt-cu13==0.8.1
|
| 124 |
+
nvidia-cutlass-dsl==4.8.0
|
| 125 |
+
nvidia-cutlass-dsl-libs-base==4.8.0
|
| 126 |
+
nvidia-cutlass-dsl-libs-core==4.8.0
|
| 127 |
+
nvidia-cutlass-dsl-libs-cu12==4.8.0
|
| 128 |
+
nvidia-cutlass-dsl-libs-cu13==4.8.0
|
| 129 |
+
nvidia-ml-py==13.615.71
|
| 130 |
+
nvidia-nccl-cu13==2.30.7
|
| 131 |
+
nvidia-nvjitlink==13.5.37
|
| 132 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 133 |
+
nvidia-nvtx==13.2.75
|
| 134 |
+
nvidia-nvvm==13.5.37
|
| 135 |
+
nvtx==0.2.15
|
| 136 |
+
omegaconf==2.3.1
|
| 137 |
+
openai==3.49.0
|
| 138 |
+
opencensus==0.11.4
|
| 139 |
+
opencensus-context==0.1.3
|
| 140 |
+
opencv-python-headless==5.0.0.93
|
| 141 |
+
opentelemetry-api==1.45.2
|
| 142 |
+
opentelemetry-exporter-http-transport==0.66b2
|
| 143 |
+
opentelemetry-exporter-otlp==1.45.2
|
| 144 |
+
opentelemetry-exporter-otlp-common==0.66b2
|
| 145 |
+
opentelemetry-exporter-otlp-proto-common==1.45.2
|
| 146 |
+
opentelemetry-exporter-otlp-proto-grpc==1.45.2
|
| 147 |
+
opentelemetry-exporter-otlp-proto-http==1.45.2
|
| 148 |
+
opentelemetry-exporter-prometheus==0.66b2
|
| 149 |
+
opentelemetry-proto==1.45.2
|
| 150 |
+
opentelemetry-sdk==1.45.2
|
| 151 |
+
opentelemetry-semantic-conventions==0.66b2
|
| 152 |
+
opentelemetry-semantic-conventions-ai==0.5.1
|
| 153 |
+
orjson==3.13.0
|
| 154 |
+
oss-harmony==0.0.12
|
| 155 |
+
outlines_core==0.2.14
|
| 156 |
+
packaging==25.0
|
| 157 |
+
pandas==3.1.1
|
| 158 |
+
partial-json-parser==0.2.1.1.post7
|
| 159 |
+
peft==0.20.0
|
| 160 |
+
pillow==13.0.0
|
| 161 |
+
pip==26.2.2
|
| 162 |
+
platformdirs==4.13.5
|
| 163 |
+
prometheus-fastapi-instrumentator==8.1.0
|
| 164 |
+
prometheus_client==0.26.0
|
| 165 |
+
propcache==0.5.4
|
| 166 |
+
proto-plus==1.29.0
|
| 167 |
+
protobuf==7.36.2
|
| 168 |
+
psutil==7.2.2
|
| 169 |
+
py-cpuinfo==9.0.0
|
| 170 |
+
py-spy==0.4.3
|
| 171 |
+
pyarrow==26.0.1
|
| 172 |
+
pyasn1==0.6.5
|
| 173 |
+
pyasn1_modules==0.4.2
|
| 174 |
+
pybase64==1.5.1
|
| 175 |
+
pybind11==3.1.0
|
| 176 |
+
pycountry==26.2.16
|
| 177 |
+
pycparser==3.12
|
| 178 |
+
pydantic==2.14.3
|
| 179 |
+
pydantic-extra-types==2.11.1
|
| 180 |
+
pydantic-settings==2.16.2
|
| 181 |
+
pydantic_core==2.50.4
|
| 182 |
+
Pygments==2.21.0
|
| 183 |
+
PyJWT==2.15.1
|
| 184 |
+
pylatexenc==2.11
|
| 185 |
+
pynvvideocodec==2.0.4
|
| 186 |
+
python-dateutil==2.9.0.post0
|
| 187 |
+
python-discovery==1.9.5
|
| 188 |
+
python-dotenv==1.2.4
|
| 189 |
+
python-json-logger==4.2.0
|
| 190 |
+
python-multipart==0.0.33
|
| 191 |
+
pyvers==0.1.0
|
| 192 |
+
PyYAML==6.0.3
|
| 193 |
+
pyzmq==27.2.0
|
| 194 |
+
quack-kernels==0.6.5
|
| 195 |
+
RapidFuzz==3.14.6
|
| 196 |
+
ray==2.60.0
|
| 197 |
+
referencing==0.37.0
|
| 198 |
+
regex==2026.11.9
|
| 199 |
+
requests==2.34.2
|
| 200 |
+
rich==15.0.0
|
| 201 |
+
rich-toolkit==0.20.6
|
| 202 |
+
rignore==0.8.1
|
| 203 |
+
rpds-py==2026.9.1
|
| 204 |
+
safetensors==0.8.0
|
| 205 |
+
scipy==1.18.1
|
| 206 |
+
sentencepiece==0.2.3
|
| 207 |
+
sentry-sdk==2.74.0
|
| 208 |
+
setproctitle==1.3.8
|
| 209 |
+
setuptools==80.10.2
|
| 210 |
+
shellingham==1.5.4
|
| 211 |
+
six==1.17.0
|
| 212 |
+
smart_open==9.0.3
|
| 213 |
+
sniffio==1.3.1
|
| 214 |
+
soupsieve==4.1
|
| 215 |
+
sse-starlette==3.7.0
|
| 216 |
+
starlette==1.10.0
|
| 217 |
+
supervisor==4.3.0
|
| 218 |
+
sympy==1.14.0
|
| 219 |
+
tabulate==0.10.0
|
| 220 |
+
tensorboard==2.21.0
|
| 221 |
+
tensorboard-data-server==0.7.2
|
| 222 |
+
tensordict==0.10.0
|
| 223 |
+
tiktoken==0.14.0
|
| 224 |
+
tilelang==0.1.12
|
| 225 |
+
tokenizers==0.22.2
|
| 226 |
+
tokenspeed-mla==2.0.3
|
| 227 |
+
tokenspeed-triton==3.9.10.post20261028
|
| 228 |
+
torch==2.14.0+cu132
|
| 229 |
+
torch_c_dlpack_ext==0.1.5
|
| 230 |
+
torchaudio==2.11.0+cpu
|
| 231 |
+
torchcodec==0.17.0+cu132
|
| 232 |
+
torchdata==0.11.0
|
| 233 |
+
torchvision==0.29.0+cu132
|
| 234 |
+
tqdm==4.70.1
|
| 235 |
+
transformers==5.10.4
|
| 236 |
+
triton==3.8.0
|
| 237 |
+
truststore==0.10.5
|
| 238 |
+
typer==0.29.1
|
| 239 |
+
typing-inspection==0.4.5
|
| 240 |
+
typing_extensions==4.16.0
|
| 241 |
+
urllib3==2.8.0
|
| 242 |
+
uvicorn==0.54.0
|
| 243 |
+
uvloop==0.23.0
|
| 244 |
+
verl==0.9.1
|
| 245 |
+
virtualenv==21.14.5
|
| 246 |
+
vllm==0.33.0
|
| 247 |
+
wandb==0.31.0
|
| 248 |
+
watchfiles==1.3.0
|
| 249 |
+
websockets==17.4
|
| 250 |
+
Werkzeug==3.1.9
|
| 251 |
+
wrapt==2.5.2
|
| 252 |
+
xgrammar==0.2.8
|
| 253 |
+
xxhash==4.1.0
|
| 254 |
+
yarl==1.25.1
|
| 255 |
+
z3-solver==4.15.4.0
|
| 256 |
+
zipp==4.1.2
|
export-report.json
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"checkpoint": "/workspace/train/runs/formal/sft-formula-v3/checkpoints/global_step_800",
|
| 3 |
+
"base": "/workspace/models/PaddleOCR-VL-1.6",
|
| 4 |
+
"weight_tensors": 620,
|
| 5 |
+
"changed_bf16_tensors": 541,
|
| 6 |
+
"changed_examples": [
|
| 7 |
+
"lm_head.weight",
|
| 8 |
+
"mlp_AR.linear_1.bias",
|
| 9 |
+
"mlp_AR.linear_1.weight",
|
| 10 |
+
"mlp_AR.linear_2.bias",
|
| 11 |
+
"mlp_AR.linear_2.weight",
|
| 12 |
+
"mlp_AR.pre_norm.bias",
|
| 13 |
+
"model.embed_tokens.weight",
|
| 14 |
+
"model.layers.0.mlp.down_proj.weight",
|
| 15 |
+
"model.layers.0.mlp.gate_proj.weight",
|
| 16 |
+
"model.layers.0.mlp.up_proj.weight",
|
| 17 |
+
"model.layers.0.self_attn.k_proj.weight",
|
| 18 |
+
"model.layers.0.self_attn.o_proj.weight"
|
| 19 |
+
],
|
| 20 |
+
"unused_parameters_from_base": [
|
| 21 |
+
"visual.vision_model.embeddings.packing_position_embedding.weight",
|
| 22 |
+
"visual.vision_model.head.attention.in_proj_bias",
|
| 23 |
+
"visual.vision_model.head.attention.in_proj_weight",
|
| 24 |
+
"visual.vision_model.head.attention.out_proj.bias",
|
| 25 |
+
"visual.vision_model.head.attention.out_proj.weight",
|
| 26 |
+
"visual.vision_model.head.layernorm.bias",
|
| 27 |
+
"visual.vision_model.head.layernorm.weight",
|
| 28 |
+
"visual.vision_model.head.mlp.fc1.bias",
|
| 29 |
+
"visual.vision_model.head.mlp.fc1.weight",
|
| 30 |
+
"visual.vision_model.head.mlp.fc2.bias",
|
| 31 |
+
"visual.vision_model.head.mlp.fc2.weight",
|
| 32 |
+
"visual.vision_model.head.probe"
|
| 33 |
+
],
|
| 34 |
+
"namespace": "original PaddleOCR-VL HF namespace; native Transformers conversion is reversible"
|
| 35 |
+
}
|
generation_config.json
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"eos_token_id": 2,
|
| 4 |
+
"pad_token_id": 0,
|
| 5 |
+
"transformers_version": "4.55.0",
|
| 6 |
+
"use_cache": false
|
| 7 |
+
}
|
image_processing_paddleocr_vl.py
ADDED
|
@@ -0,0 +1,570 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
|
| 15 |
+
"""Image processor class for PaddleOCR-VL."""
|
| 16 |
+
|
| 17 |
+
import math
|
| 18 |
+
from typing import Dict, List, Optional, Union
|
| 19 |
+
|
| 20 |
+
import numpy as np
|
| 21 |
+
import torch
|
| 22 |
+
from transformers.image_processing_utils import BaseImageProcessor, BatchFeature
|
| 23 |
+
from torchvision.transforms import functional as TF
|
| 24 |
+
from transformers.image_transforms import (
|
| 25 |
+
convert_to_rgb,
|
| 26 |
+
resize,
|
| 27 |
+
to_channel_dimension_format,
|
| 28 |
+
)
|
| 29 |
+
from transformers.image_utils import (
|
| 30 |
+
OPENAI_CLIP_MEAN,
|
| 31 |
+
OPENAI_CLIP_STD,
|
| 32 |
+
ChannelDimension,
|
| 33 |
+
PILImageResampling,
|
| 34 |
+
get_image_size,
|
| 35 |
+
infer_channel_dimension_format,
|
| 36 |
+
is_scaled_image,
|
| 37 |
+
is_valid_image,
|
| 38 |
+
make_list_of_images,
|
| 39 |
+
to_numpy_array,
|
| 40 |
+
valid_images,
|
| 41 |
+
validate_preprocess_arguments,
|
| 42 |
+
)
|
| 43 |
+
from transformers.utils import TensorType, is_vision_available, logging
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
logger = logging.get_logger(__name__)
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
if is_vision_available():
|
| 50 |
+
from PIL import Image
|
| 51 |
+
|
| 52 |
+
ImageInput = Union[
|
| 53 |
+
"PIL.Image.Image",
|
| 54 |
+
np.ndarray,
|
| 55 |
+
"torch.Tensor",
|
| 56 |
+
List["PIL.Image.Image"],
|
| 57 |
+
List[np.ndarray],
|
| 58 |
+
List["torch.Tensor"],
|
| 59 |
+
] # noqa
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
VideoInput = Union[
|
| 63 |
+
List["PIL.Image.Image"],
|
| 64 |
+
"np.ndarray",
|
| 65 |
+
"torch.Tensor",
|
| 66 |
+
List["np.ndarray"],
|
| 67 |
+
List["torch.Tensor"],
|
| 68 |
+
List[List["PIL.Image.Image"]],
|
| 69 |
+
List[List["np.ndarrray"]],
|
| 70 |
+
List[List["torch.Tensor"]],
|
| 71 |
+
] # noqa
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def make_batched_images(images) -> List[List[ImageInput]]:
|
| 75 |
+
"""
|
| 76 |
+
Accepts images in list or nested list format, and makes a list of images for preprocessing.
|
| 77 |
+
|
| 78 |
+
Args:
|
| 79 |
+
images (`Union[List[List[ImageInput]], List[ImageInput], ImageInput]`):
|
| 80 |
+
The input image.
|
| 81 |
+
|
| 82 |
+
Returns:
|
| 83 |
+
list: A list of images.
|
| 84 |
+
"""
|
| 85 |
+
if (
|
| 86 |
+
isinstance(images, (list, tuple))
|
| 87 |
+
and isinstance(images[0], (list, tuple))
|
| 88 |
+
and is_valid_image(images[0][0])
|
| 89 |
+
):
|
| 90 |
+
return [img for img_list in images for img in img_list]
|
| 91 |
+
|
| 92 |
+
elif isinstance(images, (list, tuple)) and is_valid_image(images[0]):
|
| 93 |
+
return images
|
| 94 |
+
|
| 95 |
+
elif is_valid_image(images):
|
| 96 |
+
return [images]
|
| 97 |
+
|
| 98 |
+
raise ValueError(f"Could not make batched images from {images}")
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def adjust_size(size, patch_size):
|
| 102 |
+
num_patches = size // patch_size
|
| 103 |
+
if num_patches % 2 != 0: # 如果是奇数,减1
|
| 104 |
+
num_patches -= 1
|
| 105 |
+
return num_patches * patch_size
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def make_batched_videos(videos) -> List[VideoInput]:
|
| 109 |
+
if (
|
| 110 |
+
isinstance(videos, (list, tuple))
|
| 111 |
+
and isinstance(videos[0], (list, tuple))
|
| 112 |
+
and is_valid_image(videos[0][0])
|
| 113 |
+
):
|
| 114 |
+
return videos
|
| 115 |
+
|
| 116 |
+
elif isinstance(videos, (list, tuple)) and is_valid_image(videos[0]):
|
| 117 |
+
if isinstance(videos[0], Image.Image):
|
| 118 |
+
return [videos]
|
| 119 |
+
elif len(videos[0].shape) == 4:
|
| 120 |
+
return [list(video) for video in videos]
|
| 121 |
+
|
| 122 |
+
elif is_valid_image(videos) and len(videos.shape) == 4:
|
| 123 |
+
return [list(videos)]
|
| 124 |
+
|
| 125 |
+
raise ValueError(f"Could not make batched video from {videos}")
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def smart_resize(
|
| 129 |
+
height: int,
|
| 130 |
+
width: int,
|
| 131 |
+
factor: int = 28,
|
| 132 |
+
min_pixels: int = 28 * 28 * 130,
|
| 133 |
+
max_pixels: int = 28 * 28 * 1280,
|
| 134 |
+
):
|
| 135 |
+
"""Rescales the image so that the following conditions are met:
|
| 136 |
+
|
| 137 |
+
1. Both dimensions (height and width) are divisible by 'factor'.
|
| 138 |
+
|
| 139 |
+
2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
|
| 140 |
+
|
| 141 |
+
3. The aspect ratio of the image is maintained as closely as possible.
|
| 142 |
+
|
| 143 |
+
"""
|
| 144 |
+
# if height < factor or width < factor:
|
| 145 |
+
# raise ValueError(f"height:{height} or width:{width} must be larger than factor:{factor}")
|
| 146 |
+
# if int(height < factor//4) + int(width < factor//4):
|
| 147 |
+
# raise ValueError(f"height:{height} or width:{width} must be larger than factor:{factor//4}")
|
| 148 |
+
|
| 149 |
+
if height < factor:
|
| 150 |
+
print(f"smart_resize: height={height} < factor={factor}, reset height=factor")
|
| 151 |
+
width = round((width * factor) / height)
|
| 152 |
+
height = factor
|
| 153 |
+
|
| 154 |
+
if width < factor:
|
| 155 |
+
print(f"smart_resize: width={width} < factor={factor}, reset width=factor")
|
| 156 |
+
height = round((height * factor) / width)
|
| 157 |
+
width = factor
|
| 158 |
+
|
| 159 |
+
if max(height, width) / min(height, width) > 200:
|
| 160 |
+
raise ValueError(
|
| 161 |
+
f"absolute aspect ratio must be smaller than 200, got {max(height, width) / min(height, width)}"
|
| 162 |
+
)
|
| 163 |
+
h_bar = round(height / factor) * factor
|
| 164 |
+
w_bar = round(width / factor) * factor
|
| 165 |
+
if h_bar * w_bar > max_pixels:
|
| 166 |
+
beta = math.sqrt((height * width) / max_pixels)
|
| 167 |
+
h_bar = math.floor(height / beta / factor) * factor
|
| 168 |
+
w_bar = math.floor(width / beta / factor) * factor
|
| 169 |
+
elif h_bar * w_bar < min_pixels:
|
| 170 |
+
beta = math.sqrt(min_pixels / (height * width))
|
| 171 |
+
h_bar = math.ceil(height * beta / factor) * factor
|
| 172 |
+
w_bar = math.ceil(width * beta / factor) * factor
|
| 173 |
+
return h_bar, w_bar
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
class PaddleOCRVLImageProcessor(BaseImageProcessor):
|
| 177 |
+
r"""
|
| 178 |
+
Constructs a Siglip image processor that dynamically resizes images based on the original images.
|
| 179 |
+
|
| 180 |
+
Args:
|
| 181 |
+
do_resize (`bool`, *optional*, defaults to `True`):
|
| 182 |
+
Whether to resize the image's (height, width) dimensions.
|
| 183 |
+
resample (`PILImageResampling`, *optional*, defaults to `Resampling.BICUBIC`):
|
| 184 |
+
Resampling filter to use when resizing the image.
|
| 185 |
+
do_rescale (`bool`, *optional*, defaults to `True`):
|
| 186 |
+
Whether to rescale the image by the specified scale `rescale_factor`.
|
| 187 |
+
rescale_factor (`int` or `float`, *optional*, defaults to `1/255`):
|
| 188 |
+
Scale factor to use if rescaling the image.
|
| 189 |
+
do_normalize (`bool`, *optional*, defaults to `True`):
|
| 190 |
+
Whether to normalize the image.
|
| 191 |
+
image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`):
|
| 192 |
+
Mean to use if normalizing the image. This is a float or list of floats for each channel in the image.
|
| 193 |
+
image_std (`float` or `List[float]`, *optional*, defaults to `[0.26862954, 0.26130258, 0.27577711]`):
|
| 194 |
+
Standard deviation to use if normalizing the image. This is a float or list of floats for each channel in the image.
|
| 195 |
+
do_convert_rgb (`bool`, *optional*, defaults to `True`):
|
| 196 |
+
Whether to convert the image to RGB.
|
| 197 |
+
min_pixels (`int`, *optional*, defaults to `28 * 28 * 130`):
|
| 198 |
+
The min pixels of the image to resize the image.
|
| 199 |
+
max_pixels (`int`, *optional*, defaults to `28 * 28 * 1670`):
|
| 200 |
+
The max pixels of the image to resize the image.
|
| 201 |
+
patch_size (`int`, *optional*, defaults to 14):
|
| 202 |
+
The spacial patch size of the vision encoder.
|
| 203 |
+
temporal_patch_size (`int`, *optional*, defaults to 2):
|
| 204 |
+
The temporal patch size of the vision encoder.
|
| 205 |
+
merge_size (`int`, *optional*, defaults to 2):
|
| 206 |
+
The merge size of the vision encoder to llm encoder.
|
| 207 |
+
"""
|
| 208 |
+
|
| 209 |
+
model_input_names = [
|
| 210 |
+
"pixel_values",
|
| 211 |
+
"image_grid_thw",
|
| 212 |
+
"pixel_values_videos",
|
| 213 |
+
"video_grid_thw",
|
| 214 |
+
]
|
| 215 |
+
|
| 216 |
+
def __init__(
|
| 217 |
+
self,
|
| 218 |
+
do_resize: bool = True,
|
| 219 |
+
resample: PILImageResampling = PILImageResampling.BICUBIC,
|
| 220 |
+
do_rescale: bool = True,
|
| 221 |
+
rescale_factor: Union[int, float] = 1 / 255,
|
| 222 |
+
do_normalize: bool = True,
|
| 223 |
+
image_mean: Optional[Union[float, List[float]]] = None,
|
| 224 |
+
image_std: Optional[Union[float, List[float]]] = None,
|
| 225 |
+
do_convert_rgb: bool = True,
|
| 226 |
+
min_pixels: int = 28 * 28 * 130,
|
| 227 |
+
max_pixels: int = 28 * 28 * 1280,
|
| 228 |
+
patch_size: int = 14,
|
| 229 |
+
temporal_patch_size: int = 1,
|
| 230 |
+
merge_size: int = 2,
|
| 231 |
+
**kwargs,
|
| 232 |
+
) -> None:
|
| 233 |
+
super().__init__(**kwargs)
|
| 234 |
+
self.do_resize = do_resize
|
| 235 |
+
self.resample = resample
|
| 236 |
+
self.do_rescale = do_rescale
|
| 237 |
+
self.rescale_factor = rescale_factor
|
| 238 |
+
self.do_normalize = do_normalize
|
| 239 |
+
self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN
|
| 240 |
+
self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD
|
| 241 |
+
self.min_pixels = min_pixels
|
| 242 |
+
self.max_pixels = max_pixels
|
| 243 |
+
self.patch_size = patch_size
|
| 244 |
+
self.temporal_patch_size = temporal_patch_size
|
| 245 |
+
self.merge_size = merge_size
|
| 246 |
+
self.size = {"min_pixels": min_pixels, "max_pixels": max_pixels} # not used
|
| 247 |
+
self.do_convert_rgb = do_convert_rgb
|
| 248 |
+
|
| 249 |
+
def mvit_rescale(self, image: Image.Image, merge_size: int = 2) -> Image.Image:
|
| 250 |
+
try:
|
| 251 |
+
w, h = image.size
|
| 252 |
+
except:
|
| 253 |
+
raise ValueError(str((type(image), image)))
|
| 254 |
+
patch_size = self.patch_size
|
| 255 |
+
|
| 256 |
+
if (w // patch_size) * (h // patch_size) > self.in_token_limit:
|
| 257 |
+
scale = math.sqrt(
|
| 258 |
+
self.in_token_limit / ((w // patch_size) * (h // patch_size))
|
| 259 |
+
)
|
| 260 |
+
new_w, new_h = int(w * scale), int(h * scale)
|
| 261 |
+
|
| 262 |
+
image = image.resize((new_w, new_h), Image.Resampling.BICUBIC)
|
| 263 |
+
if self.pad_input:
|
| 264 |
+
new_w, new_h = image.size
|
| 265 |
+
pad_size_h = merge_size * patch_size
|
| 266 |
+
pad_size_w = merge_size * patch_size
|
| 267 |
+
|
| 268 |
+
pad_h = (pad_size_h - new_h % pad_size_h) % pad_size_h
|
| 269 |
+
pad_w = (pad_size_w - new_w % pad_size_w) % pad_size_w
|
| 270 |
+
|
| 271 |
+
image = TF.pad(image, (0, 0, pad_w, pad_h))
|
| 272 |
+
else:
|
| 273 |
+
new_w, new_h = image.size
|
| 274 |
+
new_w = new_w - new_w % patch_size
|
| 275 |
+
new_h = new_h - new_h % patch_size
|
| 276 |
+
|
| 277 |
+
new_w = adjust_size(new_w, patch_size)
|
| 278 |
+
new_h = adjust_size(new_h, patch_size)
|
| 279 |
+
|
| 280 |
+
image = TF.center_crop(image, (new_h, new_w))
|
| 281 |
+
|
| 282 |
+
w, h = image.size
|
| 283 |
+
if w // patch_size >= 512 or h // patch_size >= 512:
|
| 284 |
+
new_h = min(patch_size * 510, h)
|
| 285 |
+
new_w = min(patch_size * 510, w)
|
| 286 |
+
image = TF.center_crop(image, (new_h, new_w))
|
| 287 |
+
# raise ValueError("Exceed pos emb")
|
| 288 |
+
return image
|
| 289 |
+
|
| 290 |
+
def _preprocess(
|
| 291 |
+
self,
|
| 292 |
+
images: Union[ImageInput, VideoInput],
|
| 293 |
+
do_resize: bool = None,
|
| 294 |
+
resample: PILImageResampling = None,
|
| 295 |
+
do_rescale: bool = None,
|
| 296 |
+
rescale_factor: float = None,
|
| 297 |
+
do_normalize: bool = None,
|
| 298 |
+
image_mean: Optional[Union[float, List[float]]] = None,
|
| 299 |
+
image_std: Optional[Union[float, List[float]]] = None,
|
| 300 |
+
do_convert_rgb: bool = None,
|
| 301 |
+
data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,
|
| 302 |
+
input_data_format: Optional[Union[str, ChannelDimension]] = None,
|
| 303 |
+
):
|
| 304 |
+
"""
|
| 305 |
+
Preprocess an image or batch of images. Copy of the `preprocess` method from `CLIPImageProcessor`.
|
| 306 |
+
|
| 307 |
+
Args:
|
| 308 |
+
images (`ImageInput`):
|
| 309 |
+
Image or batch of images to preprocess. Expects pixel values ranging from 0 to 255. If pixel values range from 0 to 1, set `do_rescale=False`.
|
| 310 |
+
vision_info (`List[Dict]`, *optional*):
|
| 311 |
+
Optional list of dictionaries containing additional information about vision inputs.
|
| 312 |
+
do_resize (`bool`, *optional*, defaults to `self.do_resize`):
|
| 313 |
+
Whether to resize the image.
|
| 314 |
+
resample (`PILImageResampling`, *optional*, defaults to `self.resample`):
|
| 315 |
+
Resampling filter to use if resizing the image. This can be one of the `PILImageResampling` enums.
|
| 316 |
+
do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
|
| 317 |
+
Whether to rescale the image.
|
| 318 |
+
rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
|
| 319 |
+
Scale factor to use if rescaling the image.
|
| 320 |
+
do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
|
| 321 |
+
Whether to normalize the image.
|
| 322 |
+
image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
|
| 323 |
+
Mean to use if normalizing the image. Can be a float or a list of floats corresponding to the number of channels in the image.
|
| 324 |
+
image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
|
| 325 |
+
Standard deviation to use if normalizing the image. Can be a float or a list of floats corresponding to the number of channels in the image.
|
| 326 |
+
do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
|
| 327 |
+
Whether to convert the image to RGB.
|
| 328 |
+
data_format (`ChannelDimension`, *optional*, defaults to `ChannelDimension.FIRST`):
|
| 329 |
+
The channel dimension format for the output image. Can be one of:
|
| 330 |
+
- `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
|
| 331 |
+
- `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
|
| 332 |
+
- Unset: Use the channel dimension format of the input image.
|
| 333 |
+
input_data_format (`ChannelDimension` or `str`, *optional*):
|
| 334 |
+
The channel dimension format for the input image. Can be one of:
|
| 335 |
+
- `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
|
| 336 |
+
- `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
|
| 337 |
+
- `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
|
| 338 |
+
"""
|
| 339 |
+
images = make_list_of_images(images)
|
| 340 |
+
|
| 341 |
+
if input_data_format is None:
|
| 342 |
+
# We assume that all images have the same channel dimension format.
|
| 343 |
+
input_data_format = ChannelDimension.LAST if isinstance(images[0], Image.Image) else infer_channel_dimension_format(images[0])
|
| 344 |
+
|
| 345 |
+
if do_convert_rgb:
|
| 346 |
+
images = [convert_to_rgb(image) for image in images]
|
| 347 |
+
|
| 348 |
+
# All transformations expect numpy arrays.
|
| 349 |
+
images = [to_numpy_array(image) for image in images]
|
| 350 |
+
|
| 351 |
+
if is_scaled_image(images[0]) and do_rescale:
|
| 352 |
+
logger.warning_once(
|
| 353 |
+
"It looks like you are trying to rescale already rescaled images. If the input"
|
| 354 |
+
" images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again."
|
| 355 |
+
)
|
| 356 |
+
|
| 357 |
+
height, width = get_image_size(images[0], channel_dim=input_data_format)
|
| 358 |
+
resized_height, resized_width = height, width
|
| 359 |
+
processed_images = []
|
| 360 |
+
|
| 361 |
+
for image in images:
|
| 362 |
+
if do_resize:
|
| 363 |
+
resized_height, resized_width = smart_resize(
|
| 364 |
+
height,
|
| 365 |
+
width,
|
| 366 |
+
factor=self.patch_size * self.merge_size,
|
| 367 |
+
min_pixels=self.min_pixels,
|
| 368 |
+
max_pixels=self.max_pixels,
|
| 369 |
+
)
|
| 370 |
+
image = resize(
|
| 371 |
+
image,
|
| 372 |
+
size=(resized_height, resized_width),
|
| 373 |
+
resample=resample,
|
| 374 |
+
input_data_format=input_data_format,
|
| 375 |
+
)
|
| 376 |
+
|
| 377 |
+
if do_rescale:
|
| 378 |
+
image = self.rescale(
|
| 379 |
+
image, scale=rescale_factor, input_data_format=input_data_format
|
| 380 |
+
)
|
| 381 |
+
|
| 382 |
+
if do_normalize:
|
| 383 |
+
image = self.normalize(
|
| 384 |
+
image=image,
|
| 385 |
+
mean=image_mean,
|
| 386 |
+
std=image_std,
|
| 387 |
+
input_data_format=input_data_format,
|
| 388 |
+
)
|
| 389 |
+
image = to_channel_dimension_format(
|
| 390 |
+
image, data_format, input_channel_dim=input_data_format
|
| 391 |
+
)
|
| 392 |
+
processed_images.append(image)
|
| 393 |
+
|
| 394 |
+
patches = np.array(processed_images)
|
| 395 |
+
if data_format == ChannelDimension.LAST:
|
| 396 |
+
patches = patches.transpose(0, 3, 1, 2)
|
| 397 |
+
if patches.shape[0] == 1:
|
| 398 |
+
patches = np.tile(patches, (self.temporal_patch_size, 1, 1, 1))
|
| 399 |
+
init_patches = patches
|
| 400 |
+
channel = patches.shape[1]
|
| 401 |
+
grid_t = patches.shape[0] // self.temporal_patch_size
|
| 402 |
+
grid_h, grid_w = (
|
| 403 |
+
resized_height // self.patch_size,
|
| 404 |
+
resized_width // self.patch_size,
|
| 405 |
+
)
|
| 406 |
+
patches = patches.reshape(
|
| 407 |
+
grid_t,
|
| 408 |
+
self.temporal_patch_size,
|
| 409 |
+
channel,
|
| 410 |
+
grid_h,
|
| 411 |
+
self.patch_size,
|
| 412 |
+
grid_w,
|
| 413 |
+
self.patch_size,
|
| 414 |
+
)
|
| 415 |
+
patches = patches.transpose(0, 3, 5, 2, 1, 4, 6)
|
| 416 |
+
assert self.temporal_patch_size == 1
|
| 417 |
+
flatten_patches = patches.reshape(
|
| 418 |
+
grid_t * grid_h * grid_w, channel, self.patch_size, self.patch_size
|
| 419 |
+
)
|
| 420 |
+
return flatten_patches, (grid_t, grid_h, grid_w)
|
| 421 |
+
|
| 422 |
+
def preprocess(
|
| 423 |
+
self,
|
| 424 |
+
images: ImageInput,
|
| 425 |
+
videos: VideoInput = None,
|
| 426 |
+
do_resize: bool = None,
|
| 427 |
+
size: Dict[str, int] = None,
|
| 428 |
+
resample: PILImageResampling = None,
|
| 429 |
+
do_rescale: bool = None,
|
| 430 |
+
rescale_factor: float = None,
|
| 431 |
+
do_normalize: bool = None,
|
| 432 |
+
image_mean: Optional[Union[float, List[float]]] = None,
|
| 433 |
+
image_std: Optional[Union[float, List[float]]] = None,
|
| 434 |
+
do_convert_rgb: bool = None,
|
| 435 |
+
return_tensors: Optional[Union[str, TensorType]] = None,
|
| 436 |
+
data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,
|
| 437 |
+
input_data_format: Optional[Union[str, ChannelDimension]] = None,
|
| 438 |
+
):
|
| 439 |
+
"""
|
| 440 |
+
Args:
|
| 441 |
+
images (`ImageInput`):
|
| 442 |
+
Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If
|
| 443 |
+
passing in images with pixel values between 0 and 1, set `do_rescale=False`.
|
| 444 |
+
videos (`VideoInput`):
|
| 445 |
+
Video to preprocess. Expects a single or batch of videos with pixel values ranging from 0 to 255. If
|
| 446 |
+
passing in videos with pixel values between 0 and 1, set `do_rescale=False`.
|
| 447 |
+
do_resize (`bool`, *optional*, defaults to `self.do_resize`):
|
| 448 |
+
Whether to resize the image.
|
| 449 |
+
size (`Dict[str, int]`, *optional*, defaults to `self.size`):
|
| 450 |
+
Size of the image after resizing. Shortest edge of the image is resized to size["shortest_edge"], with
|
| 451 |
+
the longest edge resized to keep the input aspect ratio.
|
| 452 |
+
resample (`int`, *optional*, defaults to `self.resample`):
|
| 453 |
+
Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only
|
| 454 |
+
has an effect if `do_resize` is set to `True`.
|
| 455 |
+
do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
|
| 456 |
+
Whether to rescale the image.
|
| 457 |
+
rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
|
| 458 |
+
Rescale factor to rescale the image by if `do_rescale` is set to `True`.
|
| 459 |
+
do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
|
| 460 |
+
Whether to normalize the image.
|
| 461 |
+
image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
|
| 462 |
+
Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.
|
| 463 |
+
image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
|
| 464 |
+
Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to
|
| 465 |
+
`True`.
|
| 466 |
+
do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
|
| 467 |
+
Whether to convert the image to RGB.
|
| 468 |
+
return_tensors (`str` or `TensorType`, *optional*):
|
| 469 |
+
The type of tensors to return. Can be one of:
|
| 470 |
+
- Unset: Return a list of `np.ndarray`.
|
| 471 |
+
- `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`.
|
| 472 |
+
- `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.
|
| 473 |
+
- `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.
|
| 474 |
+
- `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`.
|
| 475 |
+
data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`):
|
| 476 |
+
The channel dimension format for the output image. Can be one of:
|
| 477 |
+
- `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
|
| 478 |
+
- `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
|
| 479 |
+
- Unset: Use the channel dimension format of the input image.
|
| 480 |
+
input_data_format (`ChannelDimension` or `str`, *optional*):
|
| 481 |
+
The channel dimension format for the input image. If unset, the channel dimension format is inferred
|
| 482 |
+
from the input image. Can be one of:
|
| 483 |
+
- `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
|
| 484 |
+
- `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
|
| 485 |
+
- `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
|
| 486 |
+
|
| 487 |
+
"""
|
| 488 |
+
do_resize = do_resize if do_resize is not None else self.do_resize
|
| 489 |
+
size = size if size is not None else self.size
|
| 490 |
+
resample = resample if resample is not None else self.resample
|
| 491 |
+
do_rescale = do_rescale if do_rescale is not None else self.do_rescale
|
| 492 |
+
rescale_factor = (
|
| 493 |
+
rescale_factor if rescale_factor is not None else self.rescale_factor
|
| 494 |
+
)
|
| 495 |
+
do_normalize = do_normalize if do_normalize is not None else self.do_normalize
|
| 496 |
+
image_mean = image_mean if image_mean is not None else self.image_mean
|
| 497 |
+
image_std = image_std if image_std is not None else self.image_std
|
| 498 |
+
do_convert_rgb = (
|
| 499 |
+
do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb
|
| 500 |
+
)
|
| 501 |
+
|
| 502 |
+
if images is not None:
|
| 503 |
+
images = make_batched_images(images)
|
| 504 |
+
if videos is not None:
|
| 505 |
+
videos = make_batched_videos(videos)
|
| 506 |
+
|
| 507 |
+
if images is not None and not valid_images(images):
|
| 508 |
+
raise ValueError(
|
| 509 |
+
"Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
|
| 510 |
+
"torch.Tensor, tf.Tensor or jax.ndarray."
|
| 511 |
+
)
|
| 512 |
+
|
| 513 |
+
validate_preprocess_arguments(
|
| 514 |
+
rescale_factor=rescale_factor,
|
| 515 |
+
do_normalize=do_normalize,
|
| 516 |
+
image_mean=image_mean,
|
| 517 |
+
image_std=image_std,
|
| 518 |
+
do_resize=do_resize,
|
| 519 |
+
size=size,
|
| 520 |
+
resample=resample,
|
| 521 |
+
)
|
| 522 |
+
|
| 523 |
+
if images is not None:
|
| 524 |
+
pixel_values, vision_grid_thws = [], []
|
| 525 |
+
for image in images:
|
| 526 |
+
patches, image_grid_thw = self._preprocess(
|
| 527 |
+
image,
|
| 528 |
+
do_resize=do_resize,
|
| 529 |
+
resample=resample,
|
| 530 |
+
do_rescale=do_rescale,
|
| 531 |
+
rescale_factor=rescale_factor,
|
| 532 |
+
do_normalize=do_normalize,
|
| 533 |
+
image_mean=image_mean,
|
| 534 |
+
image_std=image_std,
|
| 535 |
+
data_format=data_format,
|
| 536 |
+
do_convert_rgb=do_convert_rgb,
|
| 537 |
+
input_data_format=input_data_format,
|
| 538 |
+
)
|
| 539 |
+
pixel_values.extend(patches)
|
| 540 |
+
vision_grid_thws.append(image_grid_thw)
|
| 541 |
+
pixel_values = np.array(pixel_values)
|
| 542 |
+
vision_grid_thws = np.array(vision_grid_thws)
|
| 543 |
+
data = {"pixel_values": pixel_values, "image_grid_thw": vision_grid_thws}
|
| 544 |
+
|
| 545 |
+
if videos is not None:
|
| 546 |
+
pixel_values, vision_grid_thws = [], []
|
| 547 |
+
for images in videos:
|
| 548 |
+
patches, video_grid_thw = self._preprocess(
|
| 549 |
+
images,
|
| 550 |
+
do_resize=do_resize,
|
| 551 |
+
resample=resample,
|
| 552 |
+
do_rescale=do_rescale,
|
| 553 |
+
rescale_factor=rescale_factor,
|
| 554 |
+
do_normalize=do_normalize,
|
| 555 |
+
image_mean=image_mean,
|
| 556 |
+
image_std=image_std,
|
| 557 |
+
data_format=data_format,
|
| 558 |
+
do_convert_rgb=do_convert_rgb,
|
| 559 |
+
input_data_format=input_data_format,
|
| 560 |
+
)
|
| 561 |
+
pixel_values.extend(patches)
|
| 562 |
+
vision_grid_thws.append(video_grid_thw)
|
| 563 |
+
pixel_values = np.array(pixel_values)
|
| 564 |
+
vision_grid_thws = np.array(vision_grid_thws)
|
| 565 |
+
data = {
|
| 566 |
+
"pixel_values_videos": pixel_values,
|
| 567 |
+
"video_grid_thw": vision_grid_thws,
|
| 568 |
+
}
|
| 569 |
+
|
| 570 |
+
return BatchFeature(data=data, tensor_type=return_tensors)
|
licenses/verl/LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Apache License
|
| 3 |
+
Version 2.0, January 2004
|
| 4 |
+
http://www.apache.org/licenses/
|
| 5 |
+
|
| 6 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 7 |
+
|
| 8 |
+
1. Definitions.
|
| 9 |
+
|
| 10 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 11 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 12 |
+
|
| 13 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 14 |
+
the copyright owner that is granting the License.
|
| 15 |
+
|
| 16 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 17 |
+
other entities that control, are controlled by, or are under common
|
| 18 |
+
control with that entity. For the purposes of this definition,
|
| 19 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 20 |
+
direction or management of such entity, whether by contract or
|
| 21 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 22 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 23 |
+
|
| 24 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 25 |
+
exercising permissions granted by this License.
|
| 26 |
+
|
| 27 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 28 |
+
including but not limited to software source code, documentation
|
| 29 |
+
source, and configuration files.
|
| 30 |
+
|
| 31 |
+
"Object" form shall mean any form resulting from mechanical
|
| 32 |
+
transformation or translation of a Source form, including but
|
| 33 |
+
not limited to compiled object code, generated documentation,
|
| 34 |
+
and conversions to other media types.
|
| 35 |
+
|
| 36 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 37 |
+
Object form, made available under the License, as indicated by a
|
| 38 |
+
copyright notice that is included in or attached to the work
|
| 39 |
+
(an example is provided in the Appendix below).
|
| 40 |
+
|
| 41 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 42 |
+
form, that is based on (or derived from) the Work and for which the
|
| 43 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 44 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 45 |
+
of this License, Derivative Works shall not include works that remain
|
| 46 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 47 |
+
the Work and Derivative Works thereof.
|
| 48 |
+
|
| 49 |
+
"Contribution" shall mean any work of authorship, including
|
| 50 |
+
the original version of the Work and any modifications or additions
|
| 51 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 52 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 53 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 54 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 55 |
+
means any form of electronic, verbal, or written communication sent
|
| 56 |
+
to the Licensor or its representatives, including but not limited to
|
| 57 |
+
communication on electronic mailing lists, source code control systems,
|
| 58 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 59 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 60 |
+
excluding communication that is conspicuously marked or otherwise
|
| 61 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 62 |
+
|
| 63 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 64 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 65 |
+
subsequently incorporated within the Work.
|
| 66 |
+
|
| 67 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 68 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 69 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 70 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 71 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 72 |
+
Work and such Derivative Works in Source or Object form.
|
| 73 |
+
|
| 74 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 75 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 76 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 77 |
+
(except as stated in this section) patent license to make, have made,
|
| 78 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 79 |
+
where such license applies only to those patent claims licensable
|
| 80 |
+
by such Contributor that are necessarily infringed by their
|
| 81 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 82 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 83 |
+
institute patent litigation against any entity (including a
|
| 84 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 85 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 86 |
+
or contributory patent infringement, then any patent licenses
|
| 87 |
+
granted to You under this License for that Work shall terminate
|
| 88 |
+
as of the date such litigation is filed.
|
| 89 |
+
|
| 90 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 91 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 92 |
+
modifications, and in Source or Object form, provided that You
|
| 93 |
+
meet the following conditions:
|
| 94 |
+
|
| 95 |
+
(a) You must give any other recipients of the Work or
|
| 96 |
+
Derivative Works a copy of this License; and
|
| 97 |
+
|
| 98 |
+
(b) You must cause any modified files to carry prominent notices
|
| 99 |
+
stating that You changed the files; and
|
| 100 |
+
|
| 101 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 102 |
+
that You distribute, all copyright, patent, trademark, and
|
| 103 |
+
attribution notices from the Source form of the Work,
|
| 104 |
+
excluding those notices that do not pertain to any part of
|
| 105 |
+
the Derivative Works; and
|
| 106 |
+
|
| 107 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 108 |
+
distribution, then any Derivative Works that You distribute must
|
| 109 |
+
include a readable copy of the attribution notices contained
|
| 110 |
+
within such NOTICE file, excluding those notices that do not
|
| 111 |
+
pertain to any part of the Derivative Works, in at least one
|
| 112 |
+
of the following places: within a NOTICE text file distributed
|
| 113 |
+
as part of the Derivative Works; within the Source form or
|
| 114 |
+
documentation, if provided along with the Derivative Works; or,
|
| 115 |
+
within a display generated by the Derivative Works, if and
|
| 116 |
+
wherever such third-party notices normally appear. The contents
|
| 117 |
+
of the NOTICE file are for informational purposes only and
|
| 118 |
+
do not modify the License. You may add Your own attribution
|
| 119 |
+
notices within Derivative Works that You distribute, alongside
|
| 120 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 121 |
+
that such additional attribution notices cannot be construed
|
| 122 |
+
as modifying the License.
|
| 123 |
+
|
| 124 |
+
You may add Your own copyright statement to Your modifications and
|
| 125 |
+
may provide additional or different license terms and conditions
|
| 126 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 127 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 128 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 129 |
+
the conditions stated in this License.
|
| 130 |
+
|
| 131 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 132 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 133 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 134 |
+
this License, without any additional terms or conditions.
|
| 135 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 136 |
+
the terms of any separate license agreement you may have executed
|
| 137 |
+
with Licensor regarding such Contributions.
|
| 138 |
+
|
| 139 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 140 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 141 |
+
except as required for reasonable and customary use in describing the
|
| 142 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 143 |
+
|
| 144 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 145 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 146 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 147 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 148 |
+
implied, including, without limitation, any warranties or conditions
|
| 149 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 150 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 151 |
+
appropriateness of using or redistributing the Work and assume any
|
| 152 |
+
risks associated with Your exercise of permissions under this License.
|
| 153 |
+
|
| 154 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 155 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 156 |
+
unless required by applicable law (such as deliberate and grossly
|
| 157 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 158 |
+
liable to You for damages, including any direct, indirect, special,
|
| 159 |
+
incidental, or consequential damages of any character arising as a
|
| 160 |
+
result of this License or out of the use or inability to use the
|
| 161 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 162 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 163 |
+
other commercial damages or losses), even if such Contributor
|
| 164 |
+
has been advised of the possibility of such damages.
|
| 165 |
+
|
| 166 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 167 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 168 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 169 |
+
or other liability obligations and/or rights consistent with this
|
| 170 |
+
License. However, in accepting such obligations, You may act only
|
| 171 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 172 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 173 |
+
defend, and hold each Contributor harmless for any liability
|
| 174 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 175 |
+
of your accepting any such warranty or additional liability.
|
| 176 |
+
|
| 177 |
+
END OF TERMS AND CONDITIONS
|
| 178 |
+
|
| 179 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 180 |
+
|
| 181 |
+
To apply the Apache License to your work, attach the following
|
| 182 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 183 |
+
replaced with your own identifying information. (Don't include
|
| 184 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 185 |
+
comment syntax for the file format. We also recommend that a
|
| 186 |
+
file or class name and description of purpose be included on the
|
| 187 |
+
same "printed page" as the copyright notice for easier
|
| 188 |
+
identification within third-party archives.
|
| 189 |
+
|
| 190 |
+
Copyright [yyyy] [name of copyright owner]
|
| 191 |
+
|
| 192 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 193 |
+
you may not use this file except in compliance with the License.
|
| 194 |
+
You may obtain a copy of the License at
|
| 195 |
+
|
| 196 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 197 |
+
|
| 198 |
+
Unless required by applicable law or agreed to in writing, software
|
| 199 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 200 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 201 |
+
See the License for the specific language governing permissions and
|
| 202 |
+
limitations under the License.
|
modeling_paddleocr_vl.py
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
preprocessor_config.json
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"auto_map": {
|
| 3 |
+
"AutoImageProcessor": "image_processing_paddleocr_vl.PaddleOCRVLImageProcessor",
|
| 4 |
+
"AutoProcessor": "processing_paddleocr_vl.PaddleOCRVLProcessor"
|
| 5 |
+
},
|
| 6 |
+
"do_convert_rgb": true,
|
| 7 |
+
"do_normalize": true,
|
| 8 |
+
"do_rescale": true,
|
| 9 |
+
"do_resize": true,
|
| 10 |
+
"image_mean": [
|
| 11 |
+
0.5,
|
| 12 |
+
0.5,
|
| 13 |
+
0.5
|
| 14 |
+
],
|
| 15 |
+
"image_processor_type": "PaddleOCRVLImageProcessor",
|
| 16 |
+
"image_std": [
|
| 17 |
+
0.5,
|
| 18 |
+
0.5,
|
| 19 |
+
0.5
|
| 20 |
+
],
|
| 21 |
+
"max_pixels": 1003520,
|
| 22 |
+
"merge_size": 2,
|
| 23 |
+
"min_pixels": 112896,
|
| 24 |
+
"patch_size": 14,
|
| 25 |
+
"processor_class": "PaddleOCRVLProcessor",
|
| 26 |
+
"resample": 3,
|
| 27 |
+
"rescale_factor": 0.00392156862745098,
|
| 28 |
+
"temporal_patch_size": 1
|
| 29 |
+
}
|
processing_paddleocr_vl.py
ADDED
|
@@ -0,0 +1,293 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
|
| 15 |
+
from typing import List, Union
|
| 16 |
+
import numpy as np
|
| 17 |
+
import torch
|
| 18 |
+
from transformers.feature_extraction_utils import BatchFeature
|
| 19 |
+
from transformers.processing_utils import (
|
| 20 |
+
ProcessingKwargs,
|
| 21 |
+
ProcessorMixin,
|
| 22 |
+
Unpack,
|
| 23 |
+
VideosKwargs,
|
| 24 |
+
)
|
| 25 |
+
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
ImageInput = Union[
|
| 29 |
+
"PIL.Image.Image",
|
| 30 |
+
np.ndarray,
|
| 31 |
+
"torch.Tensor",
|
| 32 |
+
List["PIL.Image.Image"],
|
| 33 |
+
List[np.ndarray],
|
| 34 |
+
List["torch.Tensor"],
|
| 35 |
+
] # noqa
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
VideoInput = Union[
|
| 39 |
+
List["PIL.Image.Image"],
|
| 40 |
+
"np.ndarray",
|
| 41 |
+
"torch.Tensor",
|
| 42 |
+
List["np.ndarray"],
|
| 43 |
+
List["torch.Tensor"],
|
| 44 |
+
List[List["PIL.Image.Image"]],
|
| 45 |
+
List[List["np.ndarrray"]],
|
| 46 |
+
List[List["torch.Tensor"]],
|
| 47 |
+
] # noqa
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
class PaddleOCRVLVideosProcessorKwargs(VideosKwargs, total=False):
|
| 51 |
+
fps: Union[List[float], float]
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
class PaddleOCRVLProcessorKwargs(ProcessingKwargs, total=False):
|
| 55 |
+
videos_kwargs: PaddleOCRVLVideosProcessorKwargs
|
| 56 |
+
_defaults = {
|
| 57 |
+
"text_kwargs": {
|
| 58 |
+
"padding": False,
|
| 59 |
+
},
|
| 60 |
+
"videos_kwargs": {"fps": 2.0},
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
class PaddleOCRVLProcessor(ProcessorMixin):
|
| 65 |
+
r"""
|
| 66 |
+
[`PaddleOCRVLProcessor`] offers all the functionalities of [`SiglipImageProcessor`] and [`Qwen2TokenizerFast`]. See the
|
| 67 |
+
[`~PaddleOCRVLProcessor.__call__`] and [`~PaddleOCRVLProcessor.decode`] for more information.
|
| 68 |
+
Args:
|
| 69 |
+
image_processor ([`SiglipImageProcessor`], *optional*):
|
| 70 |
+
The image processor is a required input.
|
| 71 |
+
tokenizer ([`Qwen2TokenizerFast`], *optional*):
|
| 72 |
+
The tokenizer is a required input.
|
| 73 |
+
chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages
|
| 74 |
+
in a chat into a tokenizable string.
|
| 75 |
+
"""
|
| 76 |
+
|
| 77 |
+
attributes = ["image_processor", "tokenizer"]
|
| 78 |
+
valid_kwargs = [
|
| 79 |
+
"chat_template",
|
| 80 |
+
"image_std",
|
| 81 |
+
"min_pixels",
|
| 82 |
+
"image_mean",
|
| 83 |
+
"merge_size",
|
| 84 |
+
"image_processor_type",
|
| 85 |
+
"temporal_patch_size",
|
| 86 |
+
"patch_size",
|
| 87 |
+
"max_pixels",
|
| 88 |
+
]
|
| 89 |
+
|
| 90 |
+
image_processor_class = "AutoImageProcessor"
|
| 91 |
+
tokenizer_class = "AutoTokenizer"
|
| 92 |
+
|
| 93 |
+
def __init__(
|
| 94 |
+
self, image_processor=None, tokenizer=None, chat_template=None, **kwargs
|
| 95 |
+
):
|
| 96 |
+
self.image_token = (
|
| 97 |
+
"<|IMAGE_PLACEHOLDER|>"
|
| 98 |
+
if not hasattr(tokenizer, "image_token")
|
| 99 |
+
else tokenizer.image_token
|
| 100 |
+
)
|
| 101 |
+
self.video_token = (
|
| 102 |
+
"<|video_pad|>"
|
| 103 |
+
if not hasattr(tokenizer, "video_token")
|
| 104 |
+
else tokenizer.video_token
|
| 105 |
+
)
|
| 106 |
+
super().__init__(image_processor, tokenizer, chat_template=chat_template)
|
| 107 |
+
|
| 108 |
+
def __call__(
|
| 109 |
+
self,
|
| 110 |
+
images: ImageInput = None,
|
| 111 |
+
text: Union[
|
| 112 |
+
TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]
|
| 113 |
+
] = None,
|
| 114 |
+
videos: VideoInput = None,
|
| 115 |
+
**kwargs: Unpack[PaddleOCRVLProcessorKwargs],
|
| 116 |
+
) -> BatchFeature:
|
| 117 |
+
"""
|
| 118 |
+
Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`
|
| 119 |
+
and `kwargs` arguments to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text` is not `None` to encode
|
| 120 |
+
the text. To prepare the vision inputs, this method forwards the `vision_infos` and `kwrags` arguments to
|
| 121 |
+
SiglipImageProcessor's [`~SiglipImageProcessor.__call__`] if `vision_infos` is not `None`.
|
| 122 |
+
|
| 123 |
+
Args:
|
| 124 |
+
images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`):
|
| 125 |
+
The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
|
| 126 |
+
tensor. Both channels-first and channels-last formats are supported.
|
| 127 |
+
text (`str`, `List[str]`, `List[List[str]]`):
|
| 128 |
+
The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
|
| 129 |
+
(pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
|
| 130 |
+
`is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
|
| 131 |
+
videos (`np.ndarray`, `torch.Tensor`, `List[np.ndarray]`, `List[torch.Tensor]`):
|
| 132 |
+
The image or batch of videos to be prepared. Each video can be a 4D NumPy array or PyTorch
|
| 133 |
+
tensor, or a nested list of 3D frames. Both channels-first and channels-last formats are supported.
|
| 134 |
+
return_tensors (`str` or [`~utils.TensorType`], *optional*):
|
| 135 |
+
If set, will return tensors of a particular framework. Acceptable values are:
|
| 136 |
+
- `'tf'`: Return TensorFlow `tf.constant` objects.
|
| 137 |
+
- `'pt'`: Return PyTorch `torch.Tensor` objects.
|
| 138 |
+
- `'np'`: Return NumPy `np.ndarray` objects.
|
| 139 |
+
- `'jax'`: Return JAX `jnp.ndarray` objects.
|
| 140 |
+
|
| 141 |
+
Returns:
|
| 142 |
+
[`BatchFeature`]: A [`BatchFeature`] with the following fields:
|
| 143 |
+
|
| 144 |
+
- **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
|
| 145 |
+
- **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
|
| 146 |
+
`return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
|
| 147 |
+
`None`).
|
| 148 |
+
- **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
|
| 149 |
+
- **pixel_values_videos** -- Pixel values of videos to be fed to a model. Returned when `videos` is not `None`.
|
| 150 |
+
- **image_grid_thw** -- List of image 3D grid in LLM. Returned when `images` is not `None`.
|
| 151 |
+
- **video_grid_thw** -- List of video 3D grid in LLM. Returned when `videos` is not `None`.
|
| 152 |
+
- **second_per_grid_ts** -- List of video seconds per time grid. Returned when `videos` is not `None`.
|
| 153 |
+
"""
|
| 154 |
+
output_kwargs = self._merge_kwargs(
|
| 155 |
+
PaddleOCRVLProcessorKwargs,
|
| 156 |
+
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
|
| 157 |
+
**kwargs,
|
| 158 |
+
)
|
| 159 |
+
|
| 160 |
+
if images is not None:
|
| 161 |
+
image_inputs = self.image_processor(images=images, return_tensors="pt")
|
| 162 |
+
image_inputs["pixel_values"] = image_inputs["pixel_values"]
|
| 163 |
+
image_grid_thw = image_inputs["image_grid_thw"]
|
| 164 |
+
|
| 165 |
+
else:
|
| 166 |
+
image_inputs = {}
|
| 167 |
+
image_grid_thw = None
|
| 168 |
+
|
| 169 |
+
if videos is not None:
|
| 170 |
+
# TODO: add video processing
|
| 171 |
+
videos_inputs = self.image_processor(
|
| 172 |
+
images=None, videos=videos, **output_kwargs["images_kwargs"]
|
| 173 |
+
)
|
| 174 |
+
video_grid_thw = videos_inputs["video_grid_thw"]
|
| 175 |
+
|
| 176 |
+
fps = output_kwargs["videos_kwargs"].pop("fps", 2.0)
|
| 177 |
+
if isinstance(fps, (int, float)):
|
| 178 |
+
second_per_grid_ts = [
|
| 179 |
+
self.image_processor.temporal_patch_size / fps
|
| 180 |
+
] * len(video_grid_thw)
|
| 181 |
+
elif hasattr(fps, "__len__") and len(fps) == len(video_grid_thw):
|
| 182 |
+
second_per_grid_ts = [
|
| 183 |
+
self.image_processor.temporal_patch_size / tmp for tmp in fps
|
| 184 |
+
]
|
| 185 |
+
else:
|
| 186 |
+
raise ValueError(
|
| 187 |
+
f"The length of fps ({len(fps) if hasattr(fps, '__len__') else fps}) must be equal to the length of video_grid_thw ({len(video_grid_thw)}) or fps should be a single number."
|
| 188 |
+
)
|
| 189 |
+
videos_inputs.update(
|
| 190 |
+
{"second_per_grid_ts": torch.tensor(second_per_grid_ts)}
|
| 191 |
+
)
|
| 192 |
+
|
| 193 |
+
else:
|
| 194 |
+
videos_inputs = {}
|
| 195 |
+
video_grid_thw = None
|
| 196 |
+
|
| 197 |
+
if not isinstance(text, list):
|
| 198 |
+
text = [text]
|
| 199 |
+
|
| 200 |
+
if image_grid_thw is not None:
|
| 201 |
+
index = 0
|
| 202 |
+
for i in range(len(text)):
|
| 203 |
+
while self.image_token in text[i]:
|
| 204 |
+
text[i] = text[i].replace(
|
| 205 |
+
self.image_token,
|
| 206 |
+
"<|placeholder|>"
|
| 207 |
+
* (
|
| 208 |
+
image_grid_thw[index].prod()
|
| 209 |
+
// self.image_processor.merge_size
|
| 210 |
+
// self.image_processor.merge_size
|
| 211 |
+
),
|
| 212 |
+
1,
|
| 213 |
+
)
|
| 214 |
+
index += 1
|
| 215 |
+
text[i] = text[i].replace("<|placeholder|>", self.image_token)
|
| 216 |
+
|
| 217 |
+
if video_grid_thw is not None:
|
| 218 |
+
index = 0
|
| 219 |
+
for i in range(len(text)):
|
| 220 |
+
while self.video_token in text[i]:
|
| 221 |
+
text[i] = text[i].replace(
|
| 222 |
+
self.video_token,
|
| 223 |
+
"<|placeholder|>"
|
| 224 |
+
* (
|
| 225 |
+
video_grid_thw[index].prod()
|
| 226 |
+
// self.image_processor.merge_size
|
| 227 |
+
// self.image_processor.merge_size
|
| 228 |
+
),
|
| 229 |
+
1,
|
| 230 |
+
)
|
| 231 |
+
index += 1
|
| 232 |
+
text[i] = text[i].replace("<|placeholder|>", self.video_token)
|
| 233 |
+
|
| 234 |
+
text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
|
| 235 |
+
|
| 236 |
+
return BatchFeature(data={**text_inputs, **image_inputs, **videos_inputs})
|
| 237 |
+
|
| 238 |
+
def batch_decode(self, *args, **kwargs):
|
| 239 |
+
"""
|
| 240 |
+
This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
|
| 241 |
+
refer to the docstring of this method for more information.
|
| 242 |
+
"""
|
| 243 |
+
return self.tokenizer.batch_decode(*args, **kwargs)
|
| 244 |
+
|
| 245 |
+
def decode(self, *args, **kwargs):
|
| 246 |
+
"""
|
| 247 |
+
This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
|
| 248 |
+
the docstring of this method for more information.
|
| 249 |
+
"""
|
| 250 |
+
return self.tokenizer.decode(*args, **kwargs)
|
| 251 |
+
|
| 252 |
+
def post_process_image_text_to_text(
|
| 253 |
+
self,
|
| 254 |
+
generated_outputs,
|
| 255 |
+
skip_special_tokens=True,
|
| 256 |
+
clean_up_tokenization_spaces=False,
|
| 257 |
+
**kwargs,
|
| 258 |
+
):
|
| 259 |
+
"""
|
| 260 |
+
Post-process the output of the model to decode the text.
|
| 261 |
+
|
| 262 |
+
Args:
|
| 263 |
+
generated_outputs (`torch.Tensor` or `np.ndarray`):
|
| 264 |
+
The output of the model `generate` function. The output is expected to be a tensor of shape `(batch_size, sequence_length)`
|
| 265 |
+
or `(sequence_length,)`.
|
| 266 |
+
skip_special_tokens (`bool`, *optional*, defaults to `True`):
|
| 267 |
+
Whether or not to remove special tokens in the output. Argument passed to the tokenizer's `batch_decode` method.
|
| 268 |
+
Clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
|
| 269 |
+
Whether or not to clean up the tokenization spaces. Argument passed to the tokenizer's `batch_decode` method.
|
| 270 |
+
**kwargs:
|
| 271 |
+
Additional arguments to be passed to the tokenizer's `batch_decode method`.
|
| 272 |
+
|
| 273 |
+
Returns:
|
| 274 |
+
`List[str]`: The decoded text.
|
| 275 |
+
"""
|
| 276 |
+
return self.tokenizer.batch_decode(
|
| 277 |
+
generated_outputs,
|
| 278 |
+
skip_special_tokens=skip_special_tokens,
|
| 279 |
+
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
| 280 |
+
**kwargs,
|
| 281 |
+
)
|
| 282 |
+
|
| 283 |
+
@property
|
| 284 |
+
def model_input_names(self):
|
| 285 |
+
tokenizer_input_names = self.tokenizer.model_input_names
|
| 286 |
+
image_processor_input_names = self.image_processor.model_input_names
|
| 287 |
+
names_from_processor = list(
|
| 288 |
+
dict.fromkeys(tokenizer_input_names + image_processor_input_names)
|
| 289 |
+
)
|
| 290 |
+
return names_from_processor + ["second_per_grid_ts"]
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
__all__ = ["PaddleOCRVLProcessor", "PaddleOCRVLProcessor"]
|
processor_config.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"auto_map": {
|
| 3 |
+
"AutoProcessor": "processing_paddleocr_vl.PaddleOCRVLProcessor"
|
| 4 |
+
},
|
| 5 |
+
"processor_class": "PaddleOCRVLProcessor"
|
| 6 |
+
}
|
repro/vendor/verl/workers/engine/torchtitan/utils.py
ADDED
|
@@ -0,0 +1,359 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2024 Bytedance Ltd. and/or its affiliates
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
import importlib
|
| 15 |
+
import logging
|
| 16 |
+
import re
|
| 17 |
+
from collections import defaultdict
|
| 18 |
+
from collections.abc import Generator, Iterator
|
| 19 |
+
from dataclasses import dataclass
|
| 20 |
+
from typing import Any
|
| 21 |
+
|
| 22 |
+
import torch
|
| 23 |
+
import torch.distributed
|
| 24 |
+
import torch.nn as nn
|
| 25 |
+
from torch.distributed._composable.fsdp import FSDPModule
|
| 26 |
+
from torch.distributed.tensor import DTensor
|
| 27 |
+
from torch.nn.attention.flex_attention import _mask_mod_signature, and_masks
|
| 28 |
+
from torchtitan.components.dataloader import BaseDataLoader
|
| 29 |
+
from torchtitan.models.common.attention import (
|
| 30 |
+
AttentionMasksType,
|
| 31 |
+
VarlenMetadata,
|
| 32 |
+
create_attention_mask,
|
| 33 |
+
get_causal_mask_mod,
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
logger = logging.getLogger(__name__)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class NoOpDataLoader(BaseDataLoader):
|
| 40 |
+
"""A no-op dataloader for use when verl manages its own data loading.
|
| 41 |
+
|
| 42 |
+
Satisfies the BaseDataLoader interface required by torchtitan's Trainer
|
| 43 |
+
but does nothing. Its __iter__ yields nothing, and state_dict /
|
| 44 |
+
load_state_dict are no-ops.
|
| 45 |
+
"""
|
| 46 |
+
|
| 47 |
+
@dataclass(kw_only=True, slots=True)
|
| 48 |
+
class Config(BaseDataLoader.Config):
|
| 49 |
+
pass
|
| 50 |
+
|
| 51 |
+
def __init__(self, **kwargs):
|
| 52 |
+
pass
|
| 53 |
+
|
| 54 |
+
def __iter__(self) -> Iterator[tuple[dict[str, torch.Tensor], torch.Tensor]]:
|
| 55 |
+
return iter([])
|
| 56 |
+
|
| 57 |
+
def state_dict(self):
|
| 58 |
+
return {}
|
| 59 |
+
|
| 60 |
+
def load_state_dict(self, state_dict):
|
| 61 |
+
pass
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
# Mapping from HuggingFace model_type to torchtitan model name.
|
| 65 |
+
# Torchtitan models not mapped here:
|
| 66 |
+
# - flux: diffusion model, not applicable to verl's RL/SFT workflows
|
| 67 |
+
# - llama3_ft: fault-tolerant variant of llama3, same HF models (mapped via "llama")
|
| 68 |
+
_HF_MODEL_TYPE_TO_TORCHTITAN_NAME = {
|
| 69 |
+
"qwen2": "qwen3",
|
| 70 |
+
"qwen3": "qwen3",
|
| 71 |
+
"qwen2_moe": "qwen3",
|
| 72 |
+
"qwen3_moe": "qwen3",
|
| 73 |
+
"llama": "llama3",
|
| 74 |
+
"llama4": "llama4",
|
| 75 |
+
"deepseek_v3": "deepseek_v3",
|
| 76 |
+
"gpt_oss": "gpt_oss",
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def derive_torchtitan_name_and_flavor(hf_config) -> tuple[str, str]:
|
| 81 |
+
"""Derive torchtitan model name and flavor from a HuggingFace config.
|
| 82 |
+
|
| 83 |
+
The name is mapped from ``hf_config.model_type``. The flavor is found by
|
| 84 |
+
matching architecture parameters (dim, n_layers, vocab_size) against the
|
| 85 |
+
known flavors registered in the torchtitan model package.
|
| 86 |
+
|
| 87 |
+
Args:
|
| 88 |
+
hf_config: A HuggingFace AutoConfig object.
|
| 89 |
+
|
| 90 |
+
Returns:
|
| 91 |
+
A ``(name, flavor)`` tuple.
|
| 92 |
+
|
| 93 |
+
Raises:
|
| 94 |
+
ValueError: If model_type is unsupported or no matching flavor is found.
|
| 95 |
+
"""
|
| 96 |
+
model_type = getattr(hf_config, "model_type", None)
|
| 97 |
+
if model_type is None:
|
| 98 |
+
raise ValueError("HuggingFace config does not have 'model_type' field")
|
| 99 |
+
|
| 100 |
+
name = _HF_MODEL_TYPE_TO_TORCHTITAN_NAME.get(model_type)
|
| 101 |
+
if name is None:
|
| 102 |
+
raise ValueError(
|
| 103 |
+
f"Cannot derive torchtitan model name from HF model_type '{model_type}'. "
|
| 104 |
+
f"Supported types: {list(_HF_MODEL_TYPE_TO_TORCHTITAN_NAME.keys())}."
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
# Import the model package and use model_registry to build each flavor's config.
|
| 108 |
+
# model_registry has sensible defaults for all optional params (attn_backend, etc.).
|
| 109 |
+
model_module = importlib.import_module(f"torchtitan.models.{name}")
|
| 110 |
+
model_registry = model_module.model_registry
|
| 111 |
+
|
| 112 |
+
# The configs dict name isn't derivable from the model name
|
| 113 |
+
# (e.g. gpt_oss -> gptoss_configs), so we find it by convention.
|
| 114 |
+
flavor_names = None
|
| 115 |
+
for attr, obj in vars(model_module).items():
|
| 116 |
+
if attr.endswith("_configs") and isinstance(obj, dict):
|
| 117 |
+
flavor_names = list(obj.keys())
|
| 118 |
+
break
|
| 119 |
+
|
| 120 |
+
if flavor_names is None:
|
| 121 |
+
raise ValueError(
|
| 122 |
+
f"Could not find model configs dict in torchtitan.models.{name}. "
|
| 123 |
+
f"Expected a dict attribute ending with '_configs'."
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
hidden_size = hf_config.hidden_size
|
| 127 |
+
num_layers = hf_config.num_hidden_layers
|
| 128 |
+
vocab_size = hf_config.vocab_size
|
| 129 |
+
|
| 130 |
+
for flavor_name in flavor_names:
|
| 131 |
+
cfg = model_registry(flavor_name).model
|
| 132 |
+
n_layers = getattr(cfg, "n_layers", None) or len(getattr(cfg, "layers", []))
|
| 133 |
+
if (
|
| 134 |
+
getattr(cfg, "dim", None) == hidden_size
|
| 135 |
+
and n_layers == num_layers
|
| 136 |
+
and getattr(cfg, "vocab_size", None) == vocab_size
|
| 137 |
+
):
|
| 138 |
+
logger.info(
|
| 139 |
+
f"Auto-derived torchtitan name='{name}', flavor='{flavor_name}' from HF model_type='{model_type}'"
|
| 140 |
+
)
|
| 141 |
+
return name, flavor_name
|
| 142 |
+
|
| 143 |
+
raise ValueError(
|
| 144 |
+
f"No matching torchtitan flavor found for model_type='{model_type}' "
|
| 145 |
+
f"(hidden_size={hidden_size}, num_hidden_layers={num_layers}, "
|
| 146 |
+
f"vocab_size={vocab_size}). "
|
| 147 |
+
f"Available flavors for '{name}': {flavor_names}."
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def enable_fsdp_gradient_division(model: nn.Module, dp_size: int) -> None:
|
| 152 |
+
"""
|
| 153 |
+
Re-enable FSDP's automatic gradient division.
|
| 154 |
+
|
| 155 |
+
TorchTitan calls disable_fsdp_gradient_division() which sets gradient_divide_factor=1.0.
|
| 156 |
+
This re-enables it by setting the factor to the specified dp_size, so gradients are
|
| 157 |
+
averaged across FSDP ranks. This is needed for verl's loss scaling (loss * dp_size)
|
| 158 |
+
to work correctly.
|
| 159 |
+
|
| 160 |
+
Args:
|
| 161 |
+
model: The model (or model part) to enable gradient division on.
|
| 162 |
+
dp_size: The data parallel size to use as the gradient divide factor.
|
| 163 |
+
"""
|
| 164 |
+
|
| 165 |
+
for module in model.modules():
|
| 166 |
+
if isinstance(module, FSDPModule):
|
| 167 |
+
module.set_gradient_divide_factor(float(dp_size))
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def get_attention_masks(
|
| 171 |
+
input_batch: torch.Tensor,
|
| 172 |
+
positions: torch.Tensor,
|
| 173 |
+
attn_type: str,
|
| 174 |
+
) -> AttentionMasksType:
|
| 175 |
+
match attn_type:
|
| 176 |
+
case "flex" | "flex_flash":
|
| 177 |
+
# flex_flash is FlexAttention with the FLASH kernel backend; it uses
|
| 178 |
+
# the same BlockMask as flex.
|
| 179 |
+
return _get_flex_attention_masks(
|
| 180 |
+
input_batch,
|
| 181 |
+
positions,
|
| 182 |
+
)
|
| 183 |
+
case "varlen":
|
| 184 |
+
return _create_varlen_metadata_for_document(
|
| 185 |
+
input_batch,
|
| 186 |
+
positions,
|
| 187 |
+
)
|
| 188 |
+
case _:
|
| 189 |
+
raise TypeError("Only varlen and flex attn masks are supported")
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def _get_document_mask_mod(positions: torch.Tensor) -> _mask_mod_signature:
|
| 193 |
+
# Detect boundaries from position resets
|
| 194 |
+
first_dummy_value = positions[:, :1] - 1
|
| 195 |
+
position_diff = torch.diff(positions, prepend=first_dummy_value, dim=-1)
|
| 196 |
+
sequence_indices = (position_diff != 1).cumsum(-1) # [batch, seq]
|
| 197 |
+
|
| 198 |
+
def document_mask(b: torch.Tensor, h: torch.Tensor, q_idx: torch.Tensor, kv_idx: torch.Tensor) -> torch.Tensor:
|
| 199 |
+
return sequence_indices[b, q_idx] == sequence_indices[b, kv_idx]
|
| 200 |
+
|
| 201 |
+
return document_mask
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def _get_flex_attention_masks(
|
| 205 |
+
input_batch: torch.Tensor,
|
| 206 |
+
positions: torch.Tensor,
|
| 207 |
+
) -> AttentionMasksType:
|
| 208 |
+
mask_mods = [get_causal_mask_mod()]
|
| 209 |
+
B = input_batch.shape[0]
|
| 210 |
+
mask_mods.append(_get_document_mask_mod(positions=positions))
|
| 211 |
+
return create_attention_mask(and_masks(*mask_mods), B, None, input_batch.shape[1], input_batch.shape[1])
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
def _create_varlen_metadata_for_document(input_batch: torch.Tensor, positions: torch.Tensor) -> VarlenMetadata:
|
| 215 |
+
"""
|
| 216 |
+
Creates cumulative sequence length indices needed for variable length attention
|
| 217 |
+
|
| 218 |
+
Args:
|
| 219 |
+
input_batch: Input token IDs with shape [batch, seq].
|
| 220 |
+
positions: Position IDs with shape [batch, seq]. Boundaries detected where
|
| 221 |
+
position diff != 1 (i.e., position resets).
|
| 222 |
+
|
| 223 |
+
Returns:
|
| 224 |
+
VarlenMetadata containing cumulative sequence length indices for q, k, and max_seq_len
|
| 225 |
+
"""
|
| 226 |
+
batch_size, seq_len = input_batch.shape
|
| 227 |
+
device = input_batch.device
|
| 228 |
+
|
| 229 |
+
# Detect boundaries from position resets (where diff != 1)
|
| 230 |
+
first_dummy_value = positions[:, :1] - 1
|
| 231 |
+
position_diff = torch.diff(positions, prepend=first_dummy_value, dim=-1)
|
| 232 |
+
# boundary_mask[b, i] is True if position i starts a new document
|
| 233 |
+
boundary_mask = position_diff != 1 # [batch, seq]
|
| 234 |
+
boundary_mask[:, 0] = True
|
| 235 |
+
|
| 236 |
+
cu_seqlens_list, all_seq_lengths = [], []
|
| 237 |
+
offset = 0
|
| 238 |
+
|
| 239 |
+
for b in range(batch_size):
|
| 240 |
+
# Find positions where new documents start
|
| 241 |
+
boundary_positions = boundary_mask[b].nonzero(as_tuple=True)[0].to(torch.int32)
|
| 242 |
+
sample_cu_seqlens = torch.cat(
|
| 243 |
+
[
|
| 244 |
+
boundary_positions,
|
| 245 |
+
torch.tensor([seq_len], dtype=torch.int32, device=device),
|
| 246 |
+
]
|
| 247 |
+
)
|
| 248 |
+
sample_cu_seqlens = torch.unique_consecutive(sample_cu_seqlens)
|
| 249 |
+
|
| 250 |
+
seq_lengths = torch.diff(sample_cu_seqlens)
|
| 251 |
+
all_seq_lengths.append(seq_lengths)
|
| 252 |
+
|
| 253 |
+
cu_seqlens_adjusted = sample_cu_seqlens[:-1] + offset
|
| 254 |
+
cu_seqlens_list.append(cu_seqlens_adjusted)
|
| 255 |
+
|
| 256 |
+
offset += seq_len
|
| 257 |
+
|
| 258 |
+
packed_cu_seqlens = torch.cat(cu_seqlens_list + [torch.tensor([offset], dtype=torch.int32, device=device)])
|
| 259 |
+
|
| 260 |
+
max_seqlen = 0
|
| 261 |
+
if len(all_seq_lengths) > 0:
|
| 262 |
+
all_seq_lengths = torch.cat(all_seq_lengths)
|
| 263 |
+
# device to host sync but only done once per model forward
|
| 264 |
+
max_seqlen = all_seq_lengths.max().item()
|
| 265 |
+
|
| 266 |
+
return VarlenMetadata(
|
| 267 |
+
cu_seq_q=packed_cu_seqlens,
|
| 268 |
+
cu_seq_k=packed_cu_seqlens,
|
| 269 |
+
max_q=max_seqlen,
|
| 270 |
+
max_k=max_seqlen,
|
| 271 |
+
)
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
# Regex to parse: model.layers.{L}.mlp.experts.{E}.{weight_suffix}
|
| 275 |
+
_EXPERT_PATTERN = re.compile(r"\.layers\.(\d+)\..*\.experts\.(\d+)\.(.*)")
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
def _parse_expert_name(name: str) -> tuple[int, int, str] | None:
|
| 279 |
+
"""Parse layer_id, expert_id, weight_suffix from expert param name."""
|
| 280 |
+
match = _EXPERT_PATTERN.search(name)
|
| 281 |
+
if match:
|
| 282 |
+
return int(match.group(1)), int(match.group(2)), match.group(3)
|
| 283 |
+
return None
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def _make_expert_name_template(name: str) -> str:
|
| 287 |
+
"""Convert 'model.layers.0.mlp.experts.3.w1' -> 'model.layers.0.mlp.experts.{}.w1'"""
|
| 288 |
+
return _EXPERT_PATTERN.sub(lambda m: f".layers.{m.group(1)}.mlp.experts.{{}}.{m.group(3)}", name)
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
def iter_per_tensor_params_ep(
|
| 292 |
+
params: dict[str, Any],
|
| 293 |
+
device: int,
|
| 294 |
+
ep_group: torch.distributed.ProcessGroup,
|
| 295 |
+
ep_size: int,
|
| 296 |
+
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
| 297 |
+
"""Yield (name, tensor) pairs for weight sync with Expert Parallel.
|
| 298 |
+
|
| 299 |
+
Gathers expert weights across EP ranks one (layer, weight_type) group
|
| 300 |
+
at a time to avoid OOM from materializing all experts simultaneously.
|
| 301 |
+
|
| 302 |
+
Non-expert params are yielded first (with FSDP full_tensor() if needed),
|
| 303 |
+
then expert params are all-gathered per group and yielded individually.
|
| 304 |
+
|
| 305 |
+
Args:
|
| 306 |
+
params: HF-format state dict with per-expert keys. Expert keys must
|
| 307 |
+
follow the pattern ``model.layers.{L}.mlp.experts.{E}.{suffix}``.
|
| 308 |
+
device: device ID to place tensors on.
|
| 309 |
+
ep_group: The EP process group for all-gather.
|
| 310 |
+
ep_size: Number of EP ranks.
|
| 311 |
+
"""
|
| 312 |
+
expert_params: dict[tuple[int, str], dict[int, tuple[str, Any]]] = defaultdict(dict)
|
| 313 |
+
non_expert_params: list[tuple[str, Any]] = []
|
| 314 |
+
|
| 315 |
+
for name, param in params.items():
|
| 316 |
+
parsed = _parse_expert_name(name) if "mlp.experts." in name else None
|
| 317 |
+
if parsed is None:
|
| 318 |
+
non_expert_params.append((name, param))
|
| 319 |
+
else:
|
| 320 |
+
layer_id, expert_id, weight_suffix = parsed
|
| 321 |
+
expert_params[(layer_id, weight_suffix)][expert_id] = (name, param)
|
| 322 |
+
|
| 323 |
+
params.clear()
|
| 324 |
+
|
| 325 |
+
# Yield non-expert params
|
| 326 |
+
for name, param in non_expert_params:
|
| 327 |
+
if isinstance(param, DTensor):
|
| 328 |
+
yield name, param.to(device, non_blocking=True).full_tensor().to(torch.bfloat16, non_blocking=True)
|
| 329 |
+
else:
|
| 330 |
+
yield name, param
|
| 331 |
+
del non_expert_params
|
| 332 |
+
|
| 333 |
+
# Yield expert params with all-gather
|
| 334 |
+
for (layer_id, weight_suffix), experts_dict in sorted(expert_params.items()):
|
| 335 |
+
sorted_expert_ids = sorted(experts_dict.keys())
|
| 336 |
+
|
| 337 |
+
# Stack local expert weights
|
| 338 |
+
local_weights = []
|
| 339 |
+
for eid in sorted_expert_ids:
|
| 340 |
+
_, param = experts_dict[eid]
|
| 341 |
+
if isinstance(param, DTensor):
|
| 342 |
+
param = param.to(device, non_blocking=True).full_tensor()
|
| 343 |
+
else:
|
| 344 |
+
param = param.to(device, non_blocking=True)
|
| 345 |
+
local_weights.append(param)
|
| 346 |
+
|
| 347 |
+
name_template = _make_expert_name_template(experts_dict[sorted_expert_ids[0]][0])
|
| 348 |
+
local_stacked = torch.stack(local_weights, dim=0)
|
| 349 |
+
|
| 350 |
+
# All-gather across EP ranks
|
| 351 |
+
gathered_list = [torch.empty_like(local_stacked) for _ in range(ep_size)]
|
| 352 |
+
torch.distributed.all_gather(gathered_list, local_stacked, group=ep_group)
|
| 353 |
+
all_experts = torch.cat(gathered_list, dim=0)
|
| 354 |
+
|
| 355 |
+
for expert_id in range(all_experts.shape[0]):
|
| 356 |
+
yield name_template.format(expert_id), all_experts[expert_id].to(torch.bfloat16).clone()
|
| 357 |
+
|
| 358 |
+
del local_weights, local_stacked, gathered_list, all_experts
|
| 359 |
+
torch.cuda.empty_cache()
|
repro/vendor/verl/workers/engine/veomni/transformer_impl.py
ADDED
|
@@ -0,0 +1,989 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2024 Bytedance Ltd. and/or its affiliates
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
import logging
|
| 17 |
+
from contextlib import nullcontext
|
| 18 |
+
from dataclasses import dataclass, field, fields
|
| 19 |
+
from typing import Any, Callable, Optional, Sequence
|
| 20 |
+
|
| 21 |
+
import torch
|
| 22 |
+
import torch.distributed as dist
|
| 23 |
+
from tensordict import TensorDict
|
| 24 |
+
from torch.distributed.tensor import DTensor
|
| 25 |
+
from veomni.arguments import MixedPrecisionConfig, OpsImplementationConfig
|
| 26 |
+
from veomni.distributed import parallel_state
|
| 27 |
+
from veomni.distributed.torch_parallelize import build_parallelize_model
|
| 28 |
+
from veomni.models.auto import build_foundation_model
|
| 29 |
+
from veomni.models.checkpoint_tensor_loading import get_checkpoint_tensor_converter
|
| 30 |
+
from veomni.optim import build_lr_scheduler, build_optimizer
|
| 31 |
+
from veomni.utils.seqlen_pos_transform_utils import prepare_fa_kwargs_from_position_ids
|
| 32 |
+
|
| 33 |
+
import verl.utils.torch_functional as verl_F
|
| 34 |
+
from verl.trainer.config import CheckpointConfig
|
| 35 |
+
from verl.utils import tensordict_utils as tu
|
| 36 |
+
from verl.utils.checkpoint.fsdp_checkpoint_manager import FSDPCheckpointManager
|
| 37 |
+
from verl.utils.device import get_device_id, get_device_name
|
| 38 |
+
from verl.utils.fsdp_utils import fsdp_version
|
| 39 |
+
from verl.utils.model import convert_weight_keys
|
| 40 |
+
from verl.utils.profiler import log_gpu_memory_usage
|
| 41 |
+
from verl.utils.ulysses import (
|
| 42 |
+
get_ulysses_sequence_parallel_group,
|
| 43 |
+
set_ulysses_sequence_parallel_group,
|
| 44 |
+
slice_input_tensor,
|
| 45 |
+
)
|
| 46 |
+
from verl.utils.veomni.router_replay import RouterReplayAction, VeOmniRouterReplay
|
| 47 |
+
from verl.workers.config import HFModelConfig, VeOmniEngineConfig, VeOmniOptimizerConfig
|
| 48 |
+
|
| 49 |
+
from ..base import BaseEngineCtx, EngineRegistry
|
| 50 |
+
from ..fsdp.transformer_impl import FSDPEngine, FSDPEngineWithLMHead, FSDPEngineWithValueHead
|
| 51 |
+
from ..utils import enable_full_determinism, postprocess_batch_func, prepare_micro_batches
|
| 52 |
+
from .utils import (
|
| 53 |
+
VL_TYPE2INDEX,
|
| 54 |
+
get_moe_param_handler,
|
| 55 |
+
load_safetensors_index,
|
| 56 |
+
load_veomni_model_to_gpu,
|
| 57 |
+
load_veomni_optimizer,
|
| 58 |
+
offload_veomni_model_to_cpu,
|
| 59 |
+
offload_veomni_optimizer,
|
| 60 |
+
)
|
| 61 |
+
|
| 62 |
+
logger = logging.getLogger(__file__)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _build_ops_implementation_config(engine_config: VeOmniEngineConfig) -> OpsImplementationConfig:
|
| 66 |
+
"""Forward every ``*_implementation`` selector that the installed VeOmni accepts.
|
| 67 |
+
|
| 68 |
+
``VeOmniEngineConfig`` mirrors VeOmni's ``OpsImplementationConfig`` field by field, so a
|
| 69 |
+
verl build newer than the installed VeOmni would pass unknown keyword arguments and fail
|
| 70 |
+
in the constructor. A selector the installed VeOmni does not know is skipped while it is
|
| 71 |
+
still at its verl default, and rejected otherwise, so a kernel the user explicitly asked
|
| 72 |
+
for is never silently downgraded to whatever that VeOmni version does by default.
|
| 73 |
+
"""
|
| 74 |
+
accepted = {f.name for f in fields(OpsImplementationConfig)}
|
| 75 |
+
|
| 76 |
+
kwargs, unsupported, skipped = {}, {}, []
|
| 77 |
+
for f in fields(engine_config):
|
| 78 |
+
if not f.name.endswith("_implementation"):
|
| 79 |
+
continue
|
| 80 |
+
value = getattr(engine_config, f.name)
|
| 81 |
+
if f.name in accepted:
|
| 82 |
+
kwargs[f.name] = value
|
| 83 |
+
elif value != f.default:
|
| 84 |
+
unsupported[f.name] = value
|
| 85 |
+
else:
|
| 86 |
+
skipped.append(f.name)
|
| 87 |
+
|
| 88 |
+
if unsupported:
|
| 89 |
+
raise ValueError(
|
| 90 |
+
f"The installed VeOmni's OpsImplementationConfig has no {sorted(unsupported)}, but "
|
| 91 |
+
f"they were explicitly set to {unsupported}. Upgrade VeOmni or unset these options."
|
| 92 |
+
)
|
| 93 |
+
if skipped:
|
| 94 |
+
logger.info(f"Skipping {sorted(skipped)}: unknown to the installed VeOmni, left at the verl default.")
|
| 95 |
+
|
| 96 |
+
return OpsImplementationConfig(**kwargs)
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
class VeOmniEngine(FSDPEngine):
|
| 100 |
+
_veomni_handles_position_ids = True
|
| 101 |
+
|
| 102 |
+
def _apply_veomni_input_transforms(self, model_inputs: dict, micro_batch: TensorDict):
|
| 103 |
+
"""Apply VeOmni-specific input transforms shared by LM and value heads.
|
| 104 |
+
|
| 105 |
+
Handles vision-language model masks, sequence parallel sharding,
|
| 106 |
+
and flash attention kwargs from position_ids.
|
| 107 |
+
"""
|
| 108 |
+
input_ids_rmpad = model_inputs["input_ids"]
|
| 109 |
+
sp_enabled = parallel_state.get_parallel_state().sp_enabled
|
| 110 |
+
sp_shard_collator = OmniSequenceShardCollator() if sp_enabled else None
|
| 111 |
+
|
| 112 |
+
if self.module.config.model_type in VL_TYPE2INDEX.keys():
|
| 113 |
+
image_mask = input_ids_rmpad == VL_TYPE2INDEX[self.module.config.model_type]["IMAGE_INPUT_INDEX"]
|
| 114 |
+
video_mask = input_ids_rmpad == VL_TYPE2INDEX[self.module.config.model_type]["VIDEO_INPUT_INDEX"]
|
| 115 |
+
model_inputs.update({"image_mask": image_mask, "video_mask": video_mask})
|
| 116 |
+
|
| 117 |
+
if sp_enabled:
|
| 118 |
+
sp_shard_collator(model_inputs)
|
| 119 |
+
|
| 120 |
+
use_remove_padding = tu.get_non_tensor_data(data=micro_batch, key="use_remove_padding", default=True)
|
| 121 |
+
if use_remove_padding and model_inputs.get("position_ids", None) is not None:
|
| 122 |
+
model_inputs.update(_prepare_veomni_flash_attention_kwargs(model_inputs["position_ids"]))
|
| 123 |
+
if sp_enabled:
|
| 124 |
+
model_inputs["position_ids"] = sp_shard_collator.sp_slice(model_inputs["position_ids"], dim=-1)
|
| 125 |
+
|
| 126 |
+
def __init__(
|
| 127 |
+
self,
|
| 128 |
+
model_config: HFModelConfig,
|
| 129 |
+
engine_config: VeOmniEngineConfig,
|
| 130 |
+
optimizer_config: VeOmniOptimizerConfig,
|
| 131 |
+
checkpoint_config: CheckpointConfig,
|
| 132 |
+
**kwargs,
|
| 133 |
+
):
|
| 134 |
+
"""
|
| 135 |
+
Initialize the VeOmniEngine.
|
| 136 |
+
|
| 137 |
+
Sets up distributed device meshes, LoRA, and offload policies based on config.
|
| 138 |
+
|
| 139 |
+
Args:
|
| 140 |
+
config: Configuration object with VeOmni and model settings.
|
| 141 |
+
"""
|
| 142 |
+
|
| 143 |
+
self.model_config = model_config
|
| 144 |
+
self.engine_config = engine_config
|
| 145 |
+
self.optimizer_config = optimizer_config
|
| 146 |
+
self.checkpoint_config = checkpoint_config
|
| 147 |
+
# VeOmniEngine only supports fsdp2.
|
| 148 |
+
self.data_parallel_mode = "fsdp2"
|
| 149 |
+
self.rank = dist.get_rank()
|
| 150 |
+
|
| 151 |
+
fsdp_size = self.engine_config.fsdp_size
|
| 152 |
+
world_size = dist.get_world_size()
|
| 153 |
+
dp_size = world_size // self.engine_config.ulysses_parallel_size
|
| 154 |
+
|
| 155 |
+
if fsdp_size < 0 or fsdp_size >= dp_size:
|
| 156 |
+
data_parallel_replicate_size = 1
|
| 157 |
+
data_parallel_shard_size = dp_size
|
| 158 |
+
else:
|
| 159 |
+
if dp_size % fsdp_size != 0:
|
| 160 |
+
raise ValueError(
|
| 161 |
+
f"Data parallel size ({dp_size}) must be divisible by fsdp_size ({fsdp_size}). "
|
| 162 |
+
"Please adjust your parallel configuration."
|
| 163 |
+
)
|
| 164 |
+
data_parallel_replicate_size = dp_size // fsdp_size
|
| 165 |
+
data_parallel_shard_size = fsdp_size
|
| 166 |
+
|
| 167 |
+
parallel_state.init_parallel_state(
|
| 168 |
+
dp_size=dp_size,
|
| 169 |
+
dp_replicate_size=data_parallel_replicate_size,
|
| 170 |
+
dp_shard_size=data_parallel_shard_size,
|
| 171 |
+
extra_parallel_sizes=(self.engine_config.expert_parallel_size,),
|
| 172 |
+
ulysses_size=self.engine_config.ulysses_parallel_size,
|
| 173 |
+
dp_mode=self.data_parallel_mode,
|
| 174 |
+
)
|
| 175 |
+
|
| 176 |
+
if self.engine_config.full_determinism:
|
| 177 |
+
enable_full_determinism(seed=self.engine_config.seed)
|
| 178 |
+
|
| 179 |
+
self.use_remove_padding = self.model_config.use_remove_padding
|
| 180 |
+
|
| 181 |
+
self._is_offload_param = self.engine_config.param_offload
|
| 182 |
+
self._is_offload_optimizer = self.engine_config.optimizer_offload
|
| 183 |
+
self._is_lora = self.model_config.lora_rank > 0
|
| 184 |
+
# When VeOmni parallelizes with enable_fsdp_offload, FSDP2 uses CPUOffloadPolicy and
|
| 185 |
+
# owns CPU<->accelerator placement. Manually calling model.to(device) then crashes
|
| 186 |
+
# state_dict() with a DTensor storage device mismatch (see #5995 / #6604, which fixed
|
| 187 |
+
# the FSDP engine; the VeOmni engine has the same paths and needs the same guard).
|
| 188 |
+
self._uses_fsdp2_cpu_offload_policy = self.engine_config.enable_fsdp_offload
|
| 189 |
+
|
| 190 |
+
self.use_ulysses_sp = parallel_state.get_parallel_state().sp_enabled
|
| 191 |
+
self.ulysses_sequence_parallel_size = self.engine_config.ulysses_parallel_size
|
| 192 |
+
|
| 193 |
+
if self.use_ulysses_sp:
|
| 194 |
+
self.ulysses_parallel_group = parallel_state.get_parallel_state().device_mesh["sp"].get_group()
|
| 195 |
+
else:
|
| 196 |
+
self.ulysses_parallel_group = None
|
| 197 |
+
|
| 198 |
+
if self.engine_config.entropy_from_logits_with_chunking:
|
| 199 |
+
entropy_from_logits = verl_F.entropy_from_logits_with_chunking
|
| 200 |
+
else:
|
| 201 |
+
entropy_from_logits = verl_F.entropy_from_logits
|
| 202 |
+
|
| 203 |
+
self.compute_entropy_from_logits = (
|
| 204 |
+
torch.compile(entropy_from_logits, dynamic=True)
|
| 205 |
+
if self.engine_config.use_torch_compile # use torch compile by default
|
| 206 |
+
else entropy_from_logits
|
| 207 |
+
)
|
| 208 |
+
|
| 209 |
+
# Router replay (R2 / R3) for MoE models. Controller is attached in
|
| 210 |
+
# initialize() after the model is built; here we only record intent.
|
| 211 |
+
self._router_replay_mode: str = self.engine_config.router_replay.mode
|
| 212 |
+
self.enable_routing_replay: bool = self._router_replay_mode != "disabled"
|
| 213 |
+
self._router_replay: VeOmniRouterReplay | None = None
|
| 214 |
+
if self.enable_routing_replay:
|
| 215 |
+
logger.info("VeOmniEngine: router_replay enabled, mode=%s", self._router_replay_mode)
|
| 216 |
+
|
| 217 |
+
self.pad_to_length: bool = self.engine_config.pad_to_length
|
| 218 |
+
self.pad_to_length_bucket: int = self.engine_config.pad_to_length_bucket
|
| 219 |
+
|
| 220 |
+
def initialize(self):
|
| 221 |
+
"""
|
| 222 |
+
Build the model, optimizer, and learning rate scheduler under VeOmni.
|
| 223 |
+
|
| 224 |
+
Applies device, dtype, and precision configurations, including mixed precision.
|
| 225 |
+
Sets up checkpoint manager and FLOPs counter.
|
| 226 |
+
"""
|
| 227 |
+
self._build_model_optimizer()
|
| 228 |
+
|
| 229 |
+
if self.enable_routing_replay:
|
| 230 |
+
# Defense in depth: the VeOmniActorConfig check is the primary
|
| 231 |
+
# fail-fast point and runs *before* engine init. By the time we get here,
|
| 232 |
+
# ``_build_model_optimizer()`` has already finished — this
|
| 233 |
+
# second check exists to catch direct ``VeOmniEngine``
|
| 234 |
+
# instantiation paths that bypass the worker (e.g. unit tests,
|
| 235 |
+
# standalone debug scripts) so the user gets a typed config
|
| 236 |
+
# error instead of an opaque mid-step ``AttributeError`` on
|
| 237 |
+
# ``input_ids.offsets()``.
|
| 238 |
+
if not self.engine_config.use_remove_padding:
|
| 239 |
+
raise RuntimeError(
|
| 240 |
+
"router_replay requires use_remove_padding=True. In VeOmni engine, "
|
| 241 |
+
"the non-remove-padding path also disables Ulysses SP slicing and "
|
| 242 |
+
"the fused-kernel log_probs path, and is not a tested production "
|
| 243 |
+
"configuration for MoE routing replay. Set "
|
| 244 |
+
"actor.model.use_remove_padding=True or "
|
| 245 |
+
"router_replay.mode='disabled'."
|
| 246 |
+
)
|
| 247 |
+
self._router_replay = VeOmniRouterReplay()
|
| 248 |
+
# Fails loudly if the VeOmni build in the environment does not
|
| 249 |
+
# export `set_active_replay` yet (plan requires upgrading VeOmni
|
| 250 |
+
# or disabling router_replay).
|
| 251 |
+
self._router_replay.install(self.module)
|
| 252 |
+
|
| 253 |
+
self.checkpoint_manager = FSDPCheckpointManager(
|
| 254 |
+
model=self.module,
|
| 255 |
+
optimizer=self.optimizer,
|
| 256 |
+
lr_scheduler=self.lr_scheduler,
|
| 257 |
+
processing_class=self.model_config.get_processor(),
|
| 258 |
+
checkpoint_config=self.checkpoint_config,
|
| 259 |
+
trust_remote_code=self.model_config.trust_remote_code,
|
| 260 |
+
)
|
| 261 |
+
|
| 262 |
+
self.to(
|
| 263 |
+
device="cpu",
|
| 264 |
+
model=self._is_offload_param,
|
| 265 |
+
optimizer=self._is_offload_optimizer,
|
| 266 |
+
grad=self._is_offload_optimizer,
|
| 267 |
+
)
|
| 268 |
+
|
| 269 |
+
log_gpu_memory_usage("After offload model/optimizer/grad during init", logger=logger)
|
| 270 |
+
|
| 271 |
+
def _build_optimizer(self, module):
|
| 272 |
+
optimizer = build_optimizer(
|
| 273 |
+
module,
|
| 274 |
+
lr=self.optimizer_config.lr,
|
| 275 |
+
betas=self.optimizer_config.betas,
|
| 276 |
+
weight_decay=self.optimizer_config.weight_decay,
|
| 277 |
+
optimizer_type=self.optimizer_config.optimizer,
|
| 278 |
+
)
|
| 279 |
+
get_optimizer_pre_hook = getattr(module, "get_optimizer_pre_hook", None)
|
| 280 |
+
if get_optimizer_pre_hook is not None:
|
| 281 |
+
optimizer_pre_hook = get_optimizer_pre_hook(module, module.config, self.data_parallel_mode)
|
| 282 |
+
optimizer.register_step_pre_hook(optimizer_pre_hook)
|
| 283 |
+
|
| 284 |
+
return optimizer
|
| 285 |
+
|
| 286 |
+
def _build_lr_scheduler(self, optimizer):
|
| 287 |
+
optim_config = self.optimizer_config
|
| 288 |
+
lr_scheduler = build_lr_scheduler(
|
| 289 |
+
optimizer,
|
| 290 |
+
train_steps=optim_config.total_training_steps,
|
| 291 |
+
lr=optim_config.lr,
|
| 292 |
+
lr_min=optim_config.lr_min,
|
| 293 |
+
lr_decay_style=optim_config.lr_scheduler_type,
|
| 294 |
+
lr_decay_ratio=optim_config.lr_decay_ratio,
|
| 295 |
+
lr_warmup_ratio=optim_config.lr_warmup_steps_ratio,
|
| 296 |
+
lr_start=optim_config.lr_start,
|
| 297 |
+
)
|
| 298 |
+
|
| 299 |
+
return lr_scheduler
|
| 300 |
+
|
| 301 |
+
def _get_model_config_path(self):
|
| 302 |
+
"""Return the config path (or PretrainedConfig) for build_foundation_model.
|
| 303 |
+
|
| 304 |
+
Subclasses can override to modify the HF config before model construction
|
| 305 |
+
(e.g. VeOmniEngineWithValueHead rewrites architectures to ForTokenClassification).
|
| 306 |
+
"""
|
| 307 |
+
return self.model_config.local_hf_config_path
|
| 308 |
+
|
| 309 |
+
def _maybe_apply_async_activation_offload(self, module):
|
| 310 |
+
"""Attach VeOmni's stream-based activation offload to the decoder layers.
|
| 311 |
+
|
| 312 |
+
Must run on the unparallelized module: the offload wraps each selected layer's
|
| 313 |
+
``__call__`` so that ``saved_tensors_hooks`` sits *outside* the gradient-checkpoint
|
| 314 |
+
boundary and the FSDP2 sharding that ``build_parallelize_model`` installs. Applying it
|
| 315 |
+
afterwards would put the hooks inside the checkpoint region, where the packed
|
| 316 |
+
activations are recomputed rather than saved, so nothing would be offloaded.
|
| 317 |
+
"""
|
| 318 |
+
if not self.engine_config.enable_async_activation_offload:
|
| 319 |
+
return
|
| 320 |
+
|
| 321 |
+
# Imported lazily so a VeOmni predating stream-based offload only fails when asked for it.
|
| 322 |
+
from veomni.distributed.async_offload import apply_async_activation_offload
|
| 323 |
+
|
| 324 |
+
apply_async_activation_offload(
|
| 325 |
+
module,
|
| 326 |
+
list(self.engine_config.activation_offload_modules),
|
| 327 |
+
host_cache_limit_bytes=int(self.engine_config.activation_offload_host_cache_limit_gb * 1024**3),
|
| 328 |
+
)
|
| 329 |
+
log_gpu_memory_usage("After apply async activation offload", logger=logger)
|
| 330 |
+
|
| 331 |
+
def _build_model_optimizer(self):
|
| 332 |
+
# build_foundation_model runs apply_ops_config(ops_implementation)
|
| 333 |
+
# before constructing the model, so per-model device_patch files see
|
| 334 |
+
# the resolved kernel backends.
|
| 335 |
+
ops_implementation = _build_ops_implementation_config(self.engine_config)
|
| 336 |
+
|
| 337 |
+
veomni_mixed_precision_config = MixedPrecisionConfig(enable=self.engine_config.mixed_precision)
|
| 338 |
+
|
| 339 |
+
# Load base model with specified configuration and dtype
|
| 340 |
+
module = build_foundation_model(
|
| 341 |
+
config_path=self._get_model_config_path(),
|
| 342 |
+
weights_path=self.model_config.local_path,
|
| 343 |
+
torch_dtype="float32" if veomni_mixed_precision_config.enable else "bfloat16",
|
| 344 |
+
attn_implementation=self.engine_config.attn_implementation,
|
| 345 |
+
ops_implementation=ops_implementation,
|
| 346 |
+
init_device=self.engine_config.init_device,
|
| 347 |
+
)
|
| 348 |
+
log_gpu_memory_usage("After load base model", logger=logger)
|
| 349 |
+
|
| 350 |
+
self._maybe_apply_async_activation_offload(module)
|
| 351 |
+
|
| 352 |
+
# Applies parallel strategies to the model.
|
| 353 |
+
log_gpu_memory_usage("Before parallelize model", logger=logger)
|
| 354 |
+
module = build_parallelize_model(
|
| 355 |
+
module,
|
| 356 |
+
init_device=self.engine_config.init_device,
|
| 357 |
+
weights_path=self.model_config.local_path,
|
| 358 |
+
enable_full_shard=self.engine_config.enable_full_shard,
|
| 359 |
+
mixed_precision=veomni_mixed_precision_config,
|
| 360 |
+
enable_gradient_checkpointing=self.model_config.enable_gradient_checkpointing,
|
| 361 |
+
enable_fsdp_offload=self.engine_config.enable_fsdp_offload,
|
| 362 |
+
basic_modules=list(
|
| 363 |
+
set(getattr(module, "_no_split_modules", None) or []) | set(self.engine_config.basic_modules)
|
| 364 |
+
),
|
| 365 |
+
enable_reentrant=self.engine_config.enable_reentrant,
|
| 366 |
+
enable_forward_prefetch=self.engine_config.forward_prefetch,
|
| 367 |
+
broadcast_model_weights_from_rank0=True,
|
| 368 |
+
fqn_to_index_mapping=load_safetensors_index(self.model_config.local_path),
|
| 369 |
+
)
|
| 370 |
+
log_gpu_memory_usage("After parallelize model", logger=logger)
|
| 371 |
+
|
| 372 |
+
if not self.engine_config.forward_only:
|
| 373 |
+
# Initialize optimizer with model parameters and config settings
|
| 374 |
+
optimizer = self._build_optimizer(module)
|
| 375 |
+
# Create learning rate scheduler with warmup and decay settings
|
| 376 |
+
lr_scheduler = self._build_lr_scheduler(optimizer)
|
| 377 |
+
else:
|
| 378 |
+
optimizer = None
|
| 379 |
+
lr_scheduler = None
|
| 380 |
+
|
| 381 |
+
self.module = module
|
| 382 |
+
self.optimizer = optimizer
|
| 383 |
+
self.lr_scheduler = lr_scheduler
|
| 384 |
+
|
| 385 |
+
def optimizer_step(self):
|
| 386 |
+
"""
|
| 387 |
+
Perform an optimization step using the optimizer.
|
| 388 |
+
"""
|
| 389 |
+
if hasattr(self.module, "clip_grad_norm_"):
|
| 390 |
+
grad_norm = self.module.clip_grad_norm_(self.optimizer_config.clip_grad)
|
| 391 |
+
else:
|
| 392 |
+
grad_norm = torch.nn.utils.clip_grad_norm_(self.module.parameters(), self.optimizer_config.clip_grad)
|
| 393 |
+
|
| 394 |
+
if isinstance(grad_norm, DTensor):
|
| 395 |
+
grad_norm = grad_norm.full_tensor()
|
| 396 |
+
|
| 397 |
+
# if grad_norm is not finite, skip the update
|
| 398 |
+
if not torch.isfinite(grad_norm):
|
| 399 |
+
print(f"WARN: grad_norm is not finite: {grad_norm}")
|
| 400 |
+
self.optimizer.zero_grad()
|
| 401 |
+
else:
|
| 402 |
+
self.optimizer.step()
|
| 403 |
+
return grad_norm.item()
|
| 404 |
+
|
| 405 |
+
def forward_backward_batch(self, data: TensorDict, loss_function: Callable, forward_only=False) -> Any:
|
| 406 |
+
"""
|
| 407 |
+
Perform a forward pass and optionally a backward pass on a batch of data.
|
| 408 |
+
|
| 409 |
+
Args:
|
| 410 |
+
data: The input data for the forward pass, typically containing tensors and metadata.
|
| 411 |
+
loss_function: The loss function to optimize. See `verl.workers.roles.utils.losses` for examples.
|
| 412 |
+
forward_only: If True, perform only the forward pass. If False, perform forward and backward pass.
|
| 413 |
+
|
| 414 |
+
Returns:
|
| 415 |
+
Any: The output of the forward pass, which can be used for loss computation or other purposes.
|
| 416 |
+
"""
|
| 417 |
+
tu.assign_non_tensor(data, sp_size=parallel_state.get_parallel_state().ulysses_size)
|
| 418 |
+
|
| 419 |
+
# compute num_tokens in global batch for loss normalization
|
| 420 |
+
batch_num_tokens = data["loss_mask"].sum().to(get_device_id())
|
| 421 |
+
torch.distributed.all_reduce(
|
| 422 |
+
batch_num_tokens, op=torch.distributed.ReduceOp.SUM, group=self.get_data_parallel_group()
|
| 423 |
+
)
|
| 424 |
+
tu.assign_non_tensor(data, batch_num_tokens=batch_num_tokens.item())
|
| 425 |
+
tu.assign_non_tensor(data, dp_size=self.get_data_parallel_size())
|
| 426 |
+
|
| 427 |
+
micro_batches, indices = prepare_micro_batches(
|
| 428 |
+
data=data, dp_group=self.get_data_parallel_group(), same_micro_num_in_dp=True
|
| 429 |
+
)
|
| 430 |
+
|
| 431 |
+
# Router replay state machine: decide RECORD vs REPLAY for this step.
|
| 432 |
+
# RECORD: R2 compute_log_prob (forward_only=True).
|
| 433 |
+
# REPLAY: R2 actor update, or R3 always (forward_only=True and False).
|
| 434 |
+
rr_active = self.enable_routing_replay and tu.get_non_tensor_data(data, "enable_routing_replay", default=False)
|
| 435 |
+
if rr_active:
|
| 436 |
+
assert self._router_replay is not None
|
| 437 |
+
if self._router_replay_mode == "R2" and forward_only:
|
| 438 |
+
self._router_replay.begin_record()
|
| 439 |
+
else:
|
| 440 |
+
self._router_replay.begin_replay()
|
| 441 |
+
|
| 442 |
+
ctx = torch.no_grad() if forward_only else nullcontext()
|
| 443 |
+
|
| 444 |
+
# Wrap the per-step body in try/finally so the controller is always
|
| 445 |
+
# reset to DISABLED even if forward / backward / postprocess raises.
|
| 446 |
+
# Without this, an exception leaves _recorded / _targets pinned
|
| 447 |
+
# (GPU memory) until the next successful step's begin_record/replay
|
| 448 |
+
# clears them, which may never happen if the caller (Ray actor) tears
|
| 449 |
+
# down the worker after the failure.
|
| 450 |
+
try:
|
| 451 |
+
output_lst = []
|
| 452 |
+
|
| 453 |
+
for micro_batch in micro_batches:
|
| 454 |
+
with ctx:
|
| 455 |
+
loss, meta_info = self.forward_step(
|
| 456 |
+
micro_batch, loss_function=loss_function, forward_only=forward_only
|
| 457 |
+
)
|
| 458 |
+
if not forward_only:
|
| 459 |
+
loss.backward()
|
| 460 |
+
|
| 461 |
+
output_lst.append(meta_info)
|
| 462 |
+
|
| 463 |
+
result = postprocess_batch_func(output_lst=output_lst, indices=indices, data=data)
|
| 464 |
+
return result
|
| 465 |
+
finally:
|
| 466 |
+
if rr_active:
|
| 467 |
+
self._router_replay.clear()
|
| 468 |
+
|
| 469 |
+
def get_data_parallel_rank(self):
|
| 470 |
+
return parallel_state.get_parallel_state().device_mesh.get_local_rank("dp")
|
| 471 |
+
|
| 472 |
+
def get_data_parallel_size(self):
|
| 473 |
+
return torch.distributed.get_world_size() // parallel_state.get_parallel_state().ulysses_size
|
| 474 |
+
|
| 475 |
+
def get_data_parallel_group(self):
|
| 476 |
+
if parallel_state.get_parallel_state().ulysses_size > 1:
|
| 477 |
+
return parallel_state.get_parallel_state().device_mesh.get_group(mesh_dim="dp")
|
| 478 |
+
else:
|
| 479 |
+
return torch.distributed.group.WORLD
|
| 480 |
+
|
| 481 |
+
def get_model_parallel_group(self):
|
| 482 |
+
raise NotImplementedError
|
| 483 |
+
|
| 484 |
+
def get_context_parallel_group(self):
|
| 485 |
+
raise NotImplementedError
|
| 486 |
+
|
| 487 |
+
def is_mp_src_rank_with_outputs(self):
|
| 488 |
+
"""
|
| 489 |
+
Whether the current rank is the first rank in model parallel group that contains model outputs
|
| 490 |
+
"""
|
| 491 |
+
if parallel_state.get_parallel_state().ulysses_size > 1:
|
| 492 |
+
is_collect = parallel_state.get_parallel_state().device_mesh["ulysses"].get_local_rank() == 0
|
| 493 |
+
else:
|
| 494 |
+
is_collect = True
|
| 495 |
+
return is_collect
|
| 496 |
+
|
| 497 |
+
def train_mode(self, **kwargs):
|
| 498 |
+
"""
|
| 499 |
+
Return a context manager that switches to training mode with VeOmni-specific handling.
|
| 500 |
+
|
| 501 |
+
Includes parameter and optimizer offload entry/exit.
|
| 502 |
+
"""
|
| 503 |
+
return EngineTrainModeCtx(self, **kwargs)
|
| 504 |
+
|
| 505 |
+
def eval_mode(self, **kwargs):
|
| 506 |
+
"""
|
| 507 |
+
Return a context manager that switches to evaluation mode with VeOmni-specific handling.
|
| 508 |
+
|
| 509 |
+
Includes activation offload entry/exit.
|
| 510 |
+
"""
|
| 511 |
+
return EngineEvalModeCtx(self, **kwargs)
|
| 512 |
+
|
| 513 |
+
def to(self, device: str, model: bool = True, optimizer: bool = True, grad: bool = True):
|
| 514 |
+
"""
|
| 515 |
+
Move model parameters, optimizer states, or both to the specified device.
|
| 516 |
+
Note that this function executes irrespective of offload config. It serves as manual control.
|
| 517 |
+
|
| 518 |
+
Args:
|
| 519 |
+
device: Target device identifier.
|
| 520 |
+
model: If True, move the model.
|
| 521 |
+
optimizer: If True, move the optimizer states.
|
| 522 |
+
"""
|
| 523 |
+
super(FSDPEngine, self).to(device=device, model=model, optimizer=optimizer, grad=grad)
|
| 524 |
+
|
| 525 |
+
device_name = get_device_name()
|
| 526 |
+
|
| 527 |
+
assert device in (device_name, "cpu")
|
| 528 |
+
if device == device_name:
|
| 529 |
+
if model:
|
| 530 |
+
load_veomni_model_to_gpu(self.module)
|
| 531 |
+
if optimizer and self.optimizer is not None:
|
| 532 |
+
load_veomni_optimizer(self.optimizer, device)
|
| 533 |
+
elif device == "cpu":
|
| 534 |
+
if model:
|
| 535 |
+
offload_veomni_model_to_cpu(self.module)
|
| 536 |
+
if optimizer and self.optimizer is not None:
|
| 537 |
+
offload_veomni_optimizer(self.optimizer)
|
| 538 |
+
else:
|
| 539 |
+
raise ValueError(f"Invalid device type: {device}")
|
| 540 |
+
|
| 541 |
+
def save_checkpoint(
|
| 542 |
+
self,
|
| 543 |
+
local_path: str,
|
| 544 |
+
hdfs_path: Optional[str] = None,
|
| 545 |
+
global_step: int = 0,
|
| 546 |
+
max_ckpt_to_keep: Optional[int] = None,
|
| 547 |
+
**kwargs,
|
| 548 |
+
) -> None:
|
| 549 |
+
"""
|
| 550 |
+
Save VeOmni checkpoint, handling parameter offload as needed.
|
| 551 |
+
"""
|
| 552 |
+
origin_module_device = next(self.module.parameters()).device.type
|
| 553 |
+
if (self._is_offload_param or origin_module_device == "cpu") and not getattr(
|
| 554 |
+
self, "_uses_fsdp2_cpu_offload_policy", False
|
| 555 |
+
):
|
| 556 |
+
load_veomni_model_to_gpu(self.module)
|
| 557 |
+
|
| 558 |
+
self.checkpoint_manager.save_checkpoint(
|
| 559 |
+
local_path=local_path, hdfs_path=hdfs_path, global_step=global_step, max_ckpt_to_keep=max_ckpt_to_keep
|
| 560 |
+
)
|
| 561 |
+
|
| 562 |
+
torch.distributed.barrier()
|
| 563 |
+
if self._is_offload_param:
|
| 564 |
+
offload_veomni_model_to_cpu(self.module)
|
| 565 |
+
|
| 566 |
+
def load_checkpoint(
|
| 567 |
+
self, local_path: str, hdfs_path: Optional[str] = None, del_local_after_load: int = True, **kwargs
|
| 568 |
+
) -> None:
|
| 569 |
+
"""
|
| 570 |
+
Load VeOmni checkpoint, restoring parameters and optimizer state.
|
| 571 |
+
"""
|
| 572 |
+
if self._is_offload_param and not getattr(self, "_uses_fsdp2_cpu_offload_policy", False):
|
| 573 |
+
load_veomni_model_to_gpu(self.module)
|
| 574 |
+
|
| 575 |
+
self.checkpoint_manager.load_checkpoint(
|
| 576 |
+
local_path=local_path, hdfs_path=hdfs_path, del_local_after_load=del_local_after_load
|
| 577 |
+
)
|
| 578 |
+
|
| 579 |
+
torch.distributed.barrier()
|
| 580 |
+
if self._is_offload_param:
|
| 581 |
+
offload_veomni_model_to_cpu(self.module)
|
| 582 |
+
|
| 583 |
+
if self._is_offload_optimizer:
|
| 584 |
+
offload_veomni_optimizer(self.optimizer)
|
| 585 |
+
|
| 586 |
+
def get_per_tensor_param_shard(self, **kwargs):
|
| 587 |
+
"""Yield each rank's *local* shard ``(name, local_shard, ShardSpec)`` -- the
|
| 588 |
+
DTensor export plus veomni's EP declarations. The mechanics live in
|
| 589 |
+
:func:`verl.workers.engine.veomni.utils.veomni_shard_export`; this wrapper
|
| 590 |
+
owns the offload dance (CPUOffloadPolicy manages placement itself -- see
|
| 591 |
+
#5995 -- and the delta path returns early in update_weights, so the
|
| 592 |
+
offload-back happens here, after the exporter is exhausted).
|
| 593 |
+
"""
|
| 594 |
+
from .utils import veomni_shard_export
|
| 595 |
+
|
| 596 |
+
manual_offload = not getattr(self, "_uses_fsdp2_cpu_offload_policy", False)
|
| 597 |
+
if manual_offload:
|
| 598 |
+
load_veomni_model_to_gpu(self.module)
|
| 599 |
+
gen, meta = veomni_shard_export(self.module)
|
| 600 |
+
|
| 601 |
+
def _with_offload_back():
|
| 602 |
+
yield from gen
|
| 603 |
+
if manual_offload and self._is_offload_param:
|
| 604 |
+
offload_veomni_model_to_cpu(self.module)
|
| 605 |
+
|
| 606 |
+
return _with_offload_back(), meta
|
| 607 |
+
|
| 608 |
+
def _hf_delta_entry(self, name, spec, place, lidx, lval):
|
| 609 |
+
"""veomni's per-param entry builder: EP/converter specs (fused expert
|
| 610 |
+
stacks) go through this backend's own converter machinery (see
|
| 611 |
+
:mod:`verl.workers.engine.veomni.utils`); everything else falls back to
|
| 612 |
+
the FSDP engine's DTensor identity handling."""
|
| 613 |
+
from ..spec import BlockPlacement
|
| 614 |
+
from .utils import NO_SLOTS_MSG, hf_entry_converter
|
| 615 |
+
|
| 616 |
+
if spec.to_hf_chunk is not None and isinstance(place, BlockPlacement) and spec.hf_slots is not None:
|
| 617 |
+
return hf_entry_converter(name, spec, place, lidx, lval)
|
| 618 |
+
if spec.to_hf_chunk is not None:
|
| 619 |
+
raise NotImplementedError(f"{name}: {NO_SLOTS_MSG}")
|
| 620 |
+
return super()._hf_delta_entry(name, spec, place, lidx, lval)
|
| 621 |
+
|
| 622 |
+
# get_per_tensor_param_delta_shard is inherited from FSDPEngine and
|
| 623 |
+
# prime_delta_snapshots from BaseEngine; both consume this class's
|
| 624 |
+
# get_per_tensor_param_shard and _hf_delta_entry overrides.
|
| 625 |
+
|
| 626 |
+
def get_per_tensor_param(self, **kwargs):
|
| 627 |
+
# FSDP2 CPUOffloadPolicy owns CPU<->accelerator placement; calling model.to(device)
|
| 628 |
+
# here leaves the module half-moved and crashes state_dict() below (#5995). The
|
| 629 |
+
# per-DTensor .to(device).full_tensor() in param_generator() below stages each
|
| 630 |
+
# shard instead, so the manual whole-model move is unnecessary under CPU offload.
|
| 631 |
+
if not getattr(self, "_uses_fsdp2_cpu_offload_policy", False):
|
| 632 |
+
load_veomni_model_to_gpu(self.module)
|
| 633 |
+
|
| 634 |
+
# TODO: currently only for DeepseekV4, unify all models to export weights by converter.
|
| 635 |
+
converter = get_checkpoint_tensor_converter(self.module)
|
| 636 |
+
if converter is not None and hasattr(converter, "export_weights"):
|
| 637 |
+
return converter.export_weights(self.module), None
|
| 638 |
+
|
| 639 |
+
params = self.module.state_dict()
|
| 640 |
+
params = convert_weight_keys(params, getattr(self.module, "_fsdp_wrapped_module", self.module))
|
| 641 |
+
|
| 642 |
+
if self._is_offload_param:
|
| 643 |
+
offload_veomni_model_to_cpu(self.module)
|
| 644 |
+
|
| 645 |
+
ps = parallel_state.get_parallel_state()
|
| 646 |
+
model_type = getattr(self.module.config, "model_type", "default")
|
| 647 |
+
process_func = get_moe_param_handler(model_type, ps.ep_enabled)
|
| 648 |
+
|
| 649 |
+
device = get_device_id() # used when fsdp2 set cpu_offload_policy
|
| 650 |
+
|
| 651 |
+
def param_generator():
|
| 652 |
+
for name, param in params.items():
|
| 653 |
+
unsharded_tensor = (
|
| 654 |
+
param.to(device, non_blocking=True).full_tensor() if isinstance(param, DTensor) else param
|
| 655 |
+
)
|
| 656 |
+
|
| 657 |
+
is_expert_layer = "mlp.experts." in name
|
| 658 |
+
is_proj = any(p in name for p in ["down_proj", "gate_proj", "up_proj", "gate_up_proj"])
|
| 659 |
+
|
| 660 |
+
if is_expert_layer and is_proj and ps.ep_enabled:
|
| 661 |
+
ep_rank, ep_size = ps.ep_rank, ps.ep_size
|
| 662 |
+
buffer = torch.empty_like(unsharded_tensor) # [num_experts/ep_size, H, I]
|
| 663 |
+
for src_ep_rank in range(ep_size):
|
| 664 |
+
tensor = unsharded_tensor if src_ep_rank == ep_rank else buffer
|
| 665 |
+
torch.distributed.broadcast(tensor, group_src=src_ep_rank, group=ps.ep_group)
|
| 666 |
+
yield from process_func(name, tensor, expert_id_base=src_ep_rank * tensor.size(0))
|
| 667 |
+
|
| 668 |
+
else:
|
| 669 |
+
if is_expert_layer:
|
| 670 |
+
yield from process_func(name, unsharded_tensor, expert_id_base=0)
|
| 671 |
+
else:
|
| 672 |
+
yield name, unsharded_tensor
|
| 673 |
+
|
| 674 |
+
# TODO: support VeOmni LoRA
|
| 675 |
+
return param_generator(), None
|
| 676 |
+
|
| 677 |
+
|
| 678 |
+
class EngineEvalModeCtx(BaseEngineCtx):
|
| 679 |
+
def __init__(self, engine: VeOmniEngine, **kwargs):
|
| 680 |
+
super().__init__(engine=engine, mode="eval", **kwargs)
|
| 681 |
+
|
| 682 |
+
def __enter__(self):
|
| 683 |
+
assert isinstance(self.engine, VeOmniEngine)
|
| 684 |
+
super().__enter__()
|
| 685 |
+
self.prev_sp_group = get_ulysses_sequence_parallel_group()
|
| 686 |
+
set_ulysses_sequence_parallel_group(self.engine.ulysses_parallel_group)
|
| 687 |
+
self.engine.module.train()
|
| 688 |
+
|
| 689 |
+
def __exit__(self, exc_type, exc_value, traceback):
|
| 690 |
+
assert isinstance(self.engine, VeOmniEngine)
|
| 691 |
+
set_ulysses_sequence_parallel_group(self.prev_sp_group)
|
| 692 |
+
|
| 693 |
+
# https://pytorch.org/docs/stable/notes/fsdp.html#fsdp-notes
|
| 694 |
+
# unshard the root FSDP module
|
| 695 |
+
if parallel_state.get_parallel_state().dp_shard_size > 1:
|
| 696 |
+
if fsdp_version(self.engine.module) == 1:
|
| 697 |
+
self.engine.module._handle.reshard(True)
|
| 698 |
+
elif fsdp_version(self.engine.module) == 2:
|
| 699 |
+
self.engine.module.reshard()
|
| 700 |
+
|
| 701 |
+
super().__exit__(exc_type, exc_value, traceback)
|
| 702 |
+
|
| 703 |
+
|
| 704 |
+
class EngineTrainModeCtx(BaseEngineCtx):
|
| 705 |
+
def __init__(self, engine: VeOmniEngine, **kwargs):
|
| 706 |
+
super().__init__(engine=engine, mode="train", **kwargs)
|
| 707 |
+
|
| 708 |
+
def __enter__(self):
|
| 709 |
+
assert isinstance(self.engine, VeOmniEngine)
|
| 710 |
+
super().__enter__()
|
| 711 |
+
self.prev_sp_group = get_ulysses_sequence_parallel_group()
|
| 712 |
+
set_ulysses_sequence_parallel_group(self.engine.ulysses_parallel_group)
|
| 713 |
+
# TODO: Switch to eval mode after Integrating the CI environment
|
| 714 |
+
# VeOmni (ref: https://github.com/ByteDance-Seed/VeOmni/pull/421)
|
| 715 |
+
self.engine.module.train()
|
| 716 |
+
|
| 717 |
+
def __exit__(self, exc_type, exc_value, traceback):
|
| 718 |
+
assert isinstance(self.engine, VeOmniEngine)
|
| 719 |
+
set_ulysses_sequence_parallel_group(self.prev_sp_group)
|
| 720 |
+
if self.zero_grad_on_exit or exc_type is not None:
|
| 721 |
+
self.engine.optimizer_zero_grad()
|
| 722 |
+
super().__exit__(exc_type, exc_value, traceback)
|
| 723 |
+
|
| 724 |
+
|
| 725 |
+
@dataclass
|
| 726 |
+
class OmniSequenceShardCollator:
|
| 727 |
+
"""
|
| 728 |
+
Data collator to chunk inputs along the sequence length.
|
| 729 |
+
"""
|
| 730 |
+
|
| 731 |
+
# features to slice sequence dimension
|
| 732 |
+
sp_slice_features: dict[str, int] = field(
|
| 733 |
+
default_factory=lambda: {
|
| 734 |
+
"input_ids": -1,
|
| 735 |
+
"labels": -1,
|
| 736 |
+
"pixel_values": 0,
|
| 737 |
+
"pixel_values_videos": 0,
|
| 738 |
+
},
|
| 739 |
+
metadata={"help": "features to slice sequence dimension."},
|
| 740 |
+
)
|
| 741 |
+
|
| 742 |
+
# features to padding sequence dimension
|
| 743 |
+
padding_features: dict[str, int] = field(
|
| 744 |
+
default_factory=lambda: {
|
| 745 |
+
"pixel_values": 0,
|
| 746 |
+
"pixel_values_videos": 0,
|
| 747 |
+
},
|
| 748 |
+
metadata={"help": "features to padding sequence dimension."},
|
| 749 |
+
)
|
| 750 |
+
|
| 751 |
+
# padding scale for padding features
|
| 752 |
+
padding_scale: dict[str, int] = field(
|
| 753 |
+
default_factory=lambda: {"pixel_values": 4, "pixel_values_videos": 4},
|
| 754 |
+
metadata={"help": "padding scale for padding features."},
|
| 755 |
+
)
|
| 756 |
+
|
| 757 |
+
def __post_init__(self):
|
| 758 |
+
self.sp_size = parallel_state.get_parallel_state().sp_size
|
| 759 |
+
self.sp_rank = parallel_state.get_parallel_state().sp_rank
|
| 760 |
+
|
| 761 |
+
def sp_slice(self, feature: torch.Tensor, dim: int = -1) -> dict[str, "torch.Tensor"]:
|
| 762 |
+
seq_length = feature.size(dim)
|
| 763 |
+
sp_chunk_size = (seq_length + self.sp_size - 1) // self.sp_size
|
| 764 |
+
return feature.narrow(dim, self.sp_rank * sp_chunk_size, sp_chunk_size)
|
| 765 |
+
|
| 766 |
+
def sp_padding(
|
| 767 |
+
self, tensor: "torch.Tensor", dim: int = -1, pad_value: int = 0, pad_scale: int = 1
|
| 768 |
+
) -> "torch.Tensor":
|
| 769 |
+
"""
|
| 770 |
+
Pads a tensor with pad_length to aligns tensor with sp size.
|
| 771 |
+
"""
|
| 772 |
+
seq_length = tensor.size(dim)
|
| 773 |
+
scale_sp_size = self.sp_size * pad_scale
|
| 774 |
+
|
| 775 |
+
sp_chunk_size = (seq_length + scale_sp_size - 1) // scale_sp_size
|
| 776 |
+
pad_size = sp_chunk_size * scale_sp_size - seq_length
|
| 777 |
+
if pad_size == 0:
|
| 778 |
+
return tensor
|
| 779 |
+
|
| 780 |
+
pad_shape = list(tensor.shape)
|
| 781 |
+
pad_shape[dim] = pad_size
|
| 782 |
+
pad = torch.full(pad_shape, fill_value=pad_value, dtype=tensor.dtype, device=tensor.device)
|
| 783 |
+
return torch.cat((tensor, pad), dim=dim)
|
| 784 |
+
|
| 785 |
+
def __call__(self, batch: Sequence[dict[str, "torch.Tensor"]]) -> dict[str, "torch.Tensor"]:
|
| 786 |
+
for key in batch.keys():
|
| 787 |
+
if key in self.padding_features.keys():
|
| 788 |
+
batch[key] = self.sp_padding(
|
| 789 |
+
batch[key],
|
| 790 |
+
dim=self.sp_slice_features.get(key, -1),
|
| 791 |
+
pad_value=self.padding_features[key],
|
| 792 |
+
pad_scale=self.padding_scale.get(key, 1),
|
| 793 |
+
)
|
| 794 |
+
|
| 795 |
+
# sp slice
|
| 796 |
+
for key in batch.keys():
|
| 797 |
+
if key in self.sp_slice_features.keys():
|
| 798 |
+
batch[key] = self.sp_slice(batch[key], dim=self.sp_slice_features[key])
|
| 799 |
+
|
| 800 |
+
return batch
|
| 801 |
+
|
| 802 |
+
|
| 803 |
+
def _prepare_veomni_flash_attention_kwargs(position_ids: torch.Tensor) -> dict[str, torch.Tensor | int]:
|
| 804 |
+
"""Normalize packed position_ids layout and derive varlen FlashAttention kwargs.
|
| 805 |
+
|
| 806 |
+
Supported formats for use_remove_padding=true:
|
| 807 |
+
- 2D: (1, total_nnz) - standard packed format
|
| 808 |
+
- 3D: (rope_dim, 1, total_nnz) - VeRL mRoPE packed format
|
| 809 |
+
"""
|
| 810 |
+
if position_ids.dim() == 2:
|
| 811 |
+
# (1, total_nnz) - standard packed format
|
| 812 |
+
fa_position_ids = position_ids
|
| 813 |
+
elif position_ids.dim() == 3:
|
| 814 |
+
# (rope_dim, 1, total_nnz) - VeRL mRoPE packed format
|
| 815 |
+
if position_ids.shape[1] == 1:
|
| 816 |
+
fa_position_ids = position_ids[0]
|
| 817 |
+
else:
|
| 818 |
+
raise ValueError(
|
| 819 |
+
f"Unsupported 3D position_ids shape: {tuple(position_ids.shape)}, expected (rope_dim, 1, total_nnz)"
|
| 820 |
+
)
|
| 821 |
+
else:
|
| 822 |
+
raise ValueError(
|
| 823 |
+
f"Unsupported position_ids rank: {position_ids.dim()}, "
|
| 824 |
+
f"expected 2 (1, total_nnz) or 3 (rope_dim, 1, total_nnz)"
|
| 825 |
+
)
|
| 826 |
+
|
| 827 |
+
(cu_seq_lens_q, cu_seq_lens_k), (max_length_q, max_length_k) = prepare_fa_kwargs_from_position_ids(fa_position_ids)
|
| 828 |
+
return {
|
| 829 |
+
"cu_seq_lens_q": cu_seq_lens_q,
|
| 830 |
+
"cu_seq_lens_k": cu_seq_lens_k,
|
| 831 |
+
"max_length_q": max_length_q,
|
| 832 |
+
"max_length_k": max_length_k,
|
| 833 |
+
}
|
| 834 |
+
|
| 835 |
+
|
| 836 |
+
@EngineRegistry.register(model_type="language_model", backend=["veomni"], device=["cuda", "npu"])
|
| 837 |
+
class VeOmniEngineWithLMHead(VeOmniEngine, FSDPEngineWithLMHead):
|
| 838 |
+
def prepare_model_inputs(self, micro_batch: TensorDict):
|
| 839 |
+
model_inputs, output_args = super().prepare_model_inputs(micro_batch)
|
| 840 |
+
self._apply_veomni_input_transforms(model_inputs, micro_batch)
|
| 841 |
+
|
| 842 |
+
# Activate VeOmni's chunk_logprobs path: ForCausalLMLoss short-circuits
|
| 843 |
+
# to per-token log_probs/entropy on return_log_probs=True. Pass the
|
| 844 |
+
# already-rolled labels as shift_labels so chunk_logprobs skips its
|
| 845 |
+
# internal causal shift and the output seq length matches the input —
|
| 846 |
+
# prepare_model_outputs().squeeze(0) then lands at (total_nnz,).
|
| 847 |
+
use_remove_padding = tu.get_non_tensor_data(data=micro_batch, key="use_remove_padding", default=True)
|
| 848 |
+
use_fused_kernels = tu.get_non_tensor_data(data=micro_batch, key="use_fused_kernels", default=False)
|
| 849 |
+
if use_fused_kernels and use_remove_padding:
|
| 850 |
+
input_ids_rmpad = model_inputs["input_ids"]
|
| 851 |
+
shift_labels = output_args["input_ids_rmpad_rolled"].unsqueeze(0)
|
| 852 |
+
model_inputs["labels"] = input_ids_rmpad
|
| 853 |
+
model_inputs["shift_labels"] = shift_labels
|
| 854 |
+
model_inputs["return_log_probs"] = True
|
| 855 |
+
|
| 856 |
+
# Pass teacher top-K tensors so ForCausalLMLoss routes to
|
| 857 |
+
# chunk_topk_distill_function for fused distillation. TD keys
|
| 858 |
+
# teacher_ids / teacher_logprobs are populated by verl's native
|
| 859 |
+
# distillation pipeline (see verl/trainer/distillation/losses.py).
|
| 860 |
+
distillation_use_topk = tu.get_non_tensor_data(data=micro_batch, key="distillation_use_topk", default=False)
|
| 861 |
+
if distillation_use_topk and "teacher_ids" in micro_batch.keys():
|
| 862 |
+
if "teacher_logprobs" not in micro_batch.keys():
|
| 863 |
+
raise ValueError(
|
| 864 |
+
"teacher_ids present without teacher_logprobs; "
|
| 865 |
+
"both must be provided together for fused top-K distillation."
|
| 866 |
+
)
|
| 867 |
+
# Kernel kwarg names follow veomni's chunk_topk_distill_function API.
|
| 868 |
+
teacher_topk_ids = micro_batch["teacher_ids"].values().unsqueeze(0)
|
| 869 |
+
teacher_topk_log_probs = micro_batch["teacher_logprobs"].values().unsqueeze(0)
|
| 870 |
+
# SP-slice along seqlen (dim=1); teacher tensors are 3D
|
| 871 |
+
# (1, total_nnz, K) so use slice_input_tensor directly —
|
| 872 |
+
# ulysses_pad_and_slice_inputs hardcodes 2D.
|
| 873 |
+
if self.use_ulysses_sp:
|
| 874 |
+
from verl.utils.ulysses import slice_input_tensor
|
| 875 |
+
|
| 876 |
+
teacher_topk_ids = slice_input_tensor(teacher_topk_ids, dim=1, padding=True)
|
| 877 |
+
teacher_topk_log_probs = slice_input_tensor(teacher_topk_log_probs, dim=1, padding=True)
|
| 878 |
+
model_inputs["teacher_topk_ids"] = teacher_topk_ids
|
| 879 |
+
model_inputs["teacher_topk_log_probs"] = teacher_topk_log_probs
|
| 880 |
+
|
| 881 |
+
# Arm router replay for this micro-batch. In REPLAY mode this also
|
| 882 |
+
# reshapes routed_experts with the same pad + Ulysses rule that
|
| 883 |
+
# super().prepare_model_inputs just applied to input_ids.
|
| 884 |
+
self._prepare_router_replay_inputs(micro_batch, output_args)
|
| 885 |
+
|
| 886 |
+
return model_inputs, output_args
|
| 887 |
+
|
| 888 |
+
def _prepare_router_replay_inputs(self, micro_batch: TensorDict, output_args: dict) -> None:
|
| 889 |
+
"""Arm the router-replay controller for this micro-batch.
|
| 890 |
+
|
| 891 |
+
RECORD only needs the controller reset. REPLAY additionally hands it
|
| 892 |
+
the per-layer target indices in the layout the routers will see.
|
| 893 |
+
``routed_experts`` spans the whole sequence (prompt + response) in the
|
| 894 |
+
same rmpad order as ``input_ids``, so all it takes is appending the
|
| 895 |
+
pad suffix ``super().prepare_model_inputs`` added and slicing along
|
| 896 |
+
the Ulysses SP group the same way it did.
|
| 897 |
+
"""
|
| 898 |
+
rr = self._router_replay
|
| 899 |
+
if rr is None or rr.action is RouterReplayAction.DISABLED:
|
| 900 |
+
return
|
| 901 |
+
|
| 902 |
+
if rr.action is RouterReplayAction.RECORD:
|
| 903 |
+
rr.begin_microbatch()
|
| 904 |
+
return
|
| 905 |
+
|
| 906 |
+
routed = micro_batch.get("routed_experts", None)
|
| 907 |
+
if routed is None:
|
| 908 |
+
raise RuntimeError(
|
| 909 |
+
"router_replay REPLAY: micro_batch missing 'routed_experts'. "
|
| 910 |
+
"Verify that compute_log_prob (R2) or the rollout path (R3) "
|
| 911 |
+
"attached routed_experts to the batch before this engine "
|
| 912 |
+
"call, and that left_right_2_no_padding preserved it."
|
| 913 |
+
)
|
| 914 |
+
|
| 915 |
+
# Nested-jagged [bs, seq, L, topk] -> rmpad [total_nnz, L, topk].
|
| 916 |
+
targets = (routed.values() if routed.is_nested else routed).to(torch.int64)
|
| 917 |
+
pad_size = int(output_args.get("pad_size", 0))
|
| 918 |
+
if pad_size:
|
| 919 |
+
targets = torch.cat([targets, targets.new_zeros((pad_size, *targets.shape[1:]))])
|
| 920 |
+
if self.use_ulysses_sp:
|
| 921 |
+
targets = slice_input_tensor(targets, dim=0, padding=False)
|
| 922 |
+
|
| 923 |
+
rr.begin_microbatch(targets=list(targets.unbind(dim=1)))
|
| 924 |
+
|
| 925 |
+
def prepare_model_outputs(self, output, output_args, micro_batch: TensorDict, logits_processor_func):
|
| 926 |
+
"""Attach this micro-batch's recorded MoE routing to the model output.
|
| 927 |
+
|
| 928 |
+
Mirrors how ``super()`` post-processes ``log_probs``: gather the SP
|
| 929 |
+
shards, drop the pad suffix, re-wrap per sample. From there
|
| 930 |
+
``postprocess_batch_func`` restores batch order like any other
|
| 931 |
+
per-token output.
|
| 932 |
+
"""
|
| 933 |
+
model_output = super().prepare_model_outputs(output, output_args, micro_batch, logits_processor_func)
|
| 934 |
+
|
| 935 |
+
rr = self._router_replay
|
| 936 |
+
if rr is not None and rr.action is RouterReplayAction.RECORD:
|
| 937 |
+
recorded = self._gather_and_unpad_packed(rr.take_recorded().view(torch.uint8), output_args["pad_size"])
|
| 938 |
+
model_output["routed_experts"] = torch.nested.nested_tensor_from_jagged(
|
| 939 |
+
recorded.view(torch.int16), micro_batch["input_ids"].offsets()
|
| 940 |
+
)
|
| 941 |
+
elif rr is not None and rr.action is RouterReplayAction.REPLAY:
|
| 942 |
+
if rr.num_fired != rr.num_targets:
|
| 943 |
+
raise RuntimeError(
|
| 944 |
+
f"router_replay REPLAY: {rr.num_fired} routers fired but routed_experts "
|
| 945 |
+
f"carries {rr.num_targets} layer slots. Targets are matched to routers by "
|
| 946 |
+
"fire order, so unequal counts mean every layer replayed the wrong slot. "
|
| 947 |
+
"Rollout backends index this tensor by absolute decoder-layer index, so "
|
| 948 |
+
"each decoder layer must contribute exactly one router that calls "
|
| 949 |
+
"`maybe_replay_indices` -- check that no MoE variant in this model family "
|
| 950 |
+
"(e.g. a hash-routed or dense layer) is left unhooked."
|
| 951 |
+
)
|
| 952 |
+
|
| 953 |
+
return model_output
|
| 954 |
+
|
| 955 |
+
|
| 956 |
+
@EngineRegistry.register(model_type="value_model", backend=["veomni"], device=["cuda", "npu"])
|
| 957 |
+
class VeOmniEngineWithValueHead(VeOmniEngine, FSDPEngineWithValueHead):
|
| 958 |
+
"""Value model engine using VeOmni's FSDP2 + sequence parallelism.
|
| 959 |
+
|
| 960 |
+
Combines VeOmniEngine (model init, parallel state, activation offloading)
|
| 961 |
+
with FSDPEngineWithValueHead (TokenClassification output -> per-token values).
|
| 962 |
+
"""
|
| 963 |
+
|
| 964 |
+
def _get_model_config_path(self):
|
| 965 |
+
"""Return a modified HF config that loads ForTokenClassification(num_labels=1).
|
| 966 |
+
|
| 967 |
+
Uses HF's AutoModelForTokenClassification model mapping to resolve the
|
| 968 |
+
canonical ForTokenClassification class name for this model family, then
|
| 969 |
+
sets config.architectures so VeOmni's MODELING_REGISTRY dispatches to it.
|
| 970 |
+
"""
|
| 971 |
+
from transformers import AutoModelForTokenClassification
|
| 972 |
+
from veomni.models.auto import build_config
|
| 973 |
+
|
| 974 |
+
config = build_config(self.model_config.local_hf_config_path)
|
| 975 |
+
config.num_labels = 1
|
| 976 |
+
config.classifier_dropout = 0.0
|
| 977 |
+
config.hidden_dropout = "0"
|
| 978 |
+
config.summary_dropout_prob = 0.0
|
| 979 |
+
config.tie_word_embeddings = False
|
| 980 |
+
token_cls = AutoModelForTokenClassification._model_mapping.get(type(config), None)
|
| 981 |
+
if token_cls is None:
|
| 982 |
+
raise ValueError(f"No ForTokenClassification class in transformers for {type(config).__name__}.")
|
| 983 |
+
config.architectures = [token_cls.__name__]
|
| 984 |
+
return config
|
| 985 |
+
|
| 986 |
+
def prepare_model_inputs(self, micro_batch: TensorDict):
|
| 987 |
+
model_inputs, output_args = super().prepare_model_inputs(micro_batch)
|
| 988 |
+
self._apply_veomni_input_transforms(model_inputs, micro_batch)
|
| 989 |
+
return model_inputs, output_args
|
repro/vendor/verl/workers/engine/veomni/utils.py
ADDED
|
@@ -0,0 +1,430 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2024 Bytedance Ltd. and/or its affiliates
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
import json
|
| 15 |
+
import os
|
| 16 |
+
|
| 17 |
+
import torch
|
| 18 |
+
|
| 19 |
+
from verl.utils.device import get_device_id, get_torch_device
|
| 20 |
+
from verl.workers.engine.utils import _prodshape
|
| 21 |
+
|
| 22 |
+
VL_TYPE2INDEX = {
|
| 23 |
+
"qwen2_5_vl": {
|
| 24 |
+
"IMAGE_INPUT_INDEX": 151655,
|
| 25 |
+
"VIDEO_INPUT_INDEX": 151656,
|
| 26 |
+
},
|
| 27 |
+
"qwen3_vl": {
|
| 28 |
+
"IMAGE_INPUT_INDEX": 151655,
|
| 29 |
+
"VIDEO_INPUT_INDEX": 151656,
|
| 30 |
+
},
|
| 31 |
+
"qwen3_vl_moe": {
|
| 32 |
+
"IMAGE_INPUT_INDEX": 151655,
|
| 33 |
+
"VIDEO_INPUT_INDEX": 151656,
|
| 34 |
+
},
|
| 35 |
+
"qwen3_5": {
|
| 36 |
+
"IMAGE_INPUT_INDEX": 248056,
|
| 37 |
+
"VIDEO_INPUT_INDEX": 248057,
|
| 38 |
+
},
|
| 39 |
+
"qwen3_5_moe": {
|
| 40 |
+
"IMAGE_INPUT_INDEX": 248056,
|
| 41 |
+
"VIDEO_INPUT_INDEX": 248057,
|
| 42 |
+
},
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def load_safetensors_index(model_path: str) -> dict[str, int]:
|
| 47 |
+
path = os.path.join(model_path, "model.safetensors.index.json")
|
| 48 |
+
if not os.path.exists(path):
|
| 49 |
+
return {}
|
| 50 |
+
with open(path) as f:
|
| 51 |
+
weight_map = json.load(f)["weight_map"]
|
| 52 |
+
index = {fqn: int(filename.split("-")[1]) for fqn, filename in weight_map.items()}
|
| 53 |
+
return index
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
@torch.no_grad()
|
| 57 |
+
def offload_veomni_model_to_cpu(model, empty_cache: bool = True):
|
| 58 |
+
from torch.distributed.fsdp._fully_shard._fsdp_common import TrainingState
|
| 59 |
+
from torch.distributed.fsdp._fully_shard._fsdp_state import _get_module_fsdp_state
|
| 60 |
+
|
| 61 |
+
for module in model.modules():
|
| 62 |
+
state = _get_module_fsdp_state(module)
|
| 63 |
+
if state is None:
|
| 64 |
+
continue
|
| 65 |
+
fsdp_param_group = state._fsdp_param_group
|
| 66 |
+
|
| 67 |
+
if fsdp_param_group is None:
|
| 68 |
+
continue
|
| 69 |
+
|
| 70 |
+
fsdp_param_group._training_state = TrainingState.IDLE
|
| 71 |
+
|
| 72 |
+
model.reshard()
|
| 73 |
+
model.cpu()
|
| 74 |
+
if empty_cache:
|
| 75 |
+
get_torch_device().empty_cache()
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
@torch.no_grad()
|
| 79 |
+
def load_veomni_model_to_gpu(model):
|
| 80 |
+
device = get_device_id()
|
| 81 |
+
model.to(device)
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
@torch.no_grad()
|
| 85 |
+
def offload_veomni_optimizer(optimizer):
|
| 86 |
+
optimizers = []
|
| 87 |
+
# Check if this is a MultiOptimizer (for ep and non-ep parameters when ep+fsdp2 is enabled)
|
| 88 |
+
if hasattr(optimizer, "_is_multi_optimizer") and optimizer._is_multi_optimizer:
|
| 89 |
+
optimizers.extend(optimizer.optimizers_dict.values())
|
| 90 |
+
else:
|
| 91 |
+
optimizers.append(optimizer)
|
| 92 |
+
|
| 93 |
+
for opt in optimizers:
|
| 94 |
+
if not opt.state:
|
| 95 |
+
continue
|
| 96 |
+
for param_group in opt.param_groups:
|
| 97 |
+
for param in param_group["params"]:
|
| 98 |
+
state = opt.state[param]
|
| 99 |
+
for key, value in state.items():
|
| 100 |
+
if isinstance(value, torch.Tensor):
|
| 101 |
+
state[key] = value.to("cpu", non_blocking=True)
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
@torch.no_grad()
|
| 105 |
+
def load_veomni_optimizer(optimizer, device_id):
|
| 106 |
+
optimizers = []
|
| 107 |
+
# Check if this is a MultiOptimizer (for ep and non-ep parameters when ep+fsdp2 is enabled)
|
| 108 |
+
if hasattr(optimizer, "_is_multi_optimizer") and optimizer._is_multi_optimizer:
|
| 109 |
+
optimizers.extend(optimizer.optimizers_dict.values())
|
| 110 |
+
else:
|
| 111 |
+
optimizers.append(optimizer)
|
| 112 |
+
|
| 113 |
+
for opt in optimizers:
|
| 114 |
+
if not opt.state:
|
| 115 |
+
continue
|
| 116 |
+
for param_group in opt.param_groups:
|
| 117 |
+
for param in param_group["params"]:
|
| 118 |
+
state = opt.state[param]
|
| 119 |
+
for key, value in state.items():
|
| 120 |
+
if isinstance(value, torch.Tensor):
|
| 121 |
+
state[key] = value.to(device_id, non_blocking=True)
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def _map_moe_params_common(name, tensor, expert_id_base):
|
| 125 |
+
for i in range(tensor.size(0)):
|
| 126 |
+
idx = expert_id_base + i
|
| 127 |
+
new_key = name.replace("mlp.experts.", f"mlp.experts.{idx}.") + ".weight"
|
| 128 |
+
yield new_key, tensor[i].to(get_device_id(), non_blocking=True)
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def default_moe_param_handler(name, tensor, expert_id_base):
|
| 132 |
+
"""Project a stacked expert tensor ``[k, ...]`` (dim 0 = a contiguous run of
|
| 133 |
+
experts) onto per-expert HF-named tensors. ``expert_id_base`` is the global
|
| 134 |
+
expert id of slice 0: an ep rank's local stack passes
|
| 135 |
+
``ep_rank * experts_per_rank``; an already-global stack passes ``0``."""
|
| 136 |
+
if "gate_up_proj" in name:
|
| 137 |
+
gate, up = tensor.chunk(2, dim=1)
|
| 138 |
+
params = {
|
| 139 |
+
name.replace("gate_up_proj", "gate_proj"): gate,
|
| 140 |
+
name.replace("gate_up_proj", "up_proj"): up,
|
| 141 |
+
}
|
| 142 |
+
else:
|
| 143 |
+
params = {name: tensor}
|
| 144 |
+
|
| 145 |
+
for key, value in params.items():
|
| 146 |
+
yield from _map_moe_params_common(key, value, expert_id_base)
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def passthrough_moe_param_handler(name, tensor, expert_id_base):
|
| 150 |
+
"""Keep a fused MoE checkpoint tensor unchanged."""
|
| 151 |
+
del expert_id_base
|
| 152 |
+
yield name, tensor
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def streamed_gpt_oss_moe_param_handler(name, tensor, expert_id_base):
|
| 156 |
+
"""Export GPT-OSS fused projections one global expert at a time.
|
| 157 |
+
|
| 158 |
+
The vLLM GPT-OSS loader consumes the original checkpoint orientation and
|
| 159 |
+
interleaved gate/up layout, so this handler only inserts the global expert
|
| 160 |
+
id into the key and selects the corresponding row. In particular, these
|
| 161 |
+
fused checkpoint keys do not have a trailing ``.weight`` component.
|
| 162 |
+
"""
|
| 163 |
+
for local_expert_id in range(tensor.size(0)):
|
| 164 |
+
expert_id = expert_id_base + local_expert_id
|
| 165 |
+
expert_name = name.replace("mlp.experts.", f"mlp.experts.{expert_id}.", 1)
|
| 166 |
+
yield expert_name, tensor[local_expert_id].to(get_device_id(), non_blocking=True)
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
# Overrides the default MoE parameter mapping per model_type. Handlers follow the
|
| 170 |
+
# ``default_moe_param_handler`` contract: (name, stacked_tensor, expert_id_base).
|
| 171 |
+
MOE_PARAM_HANDERS = {"gpt_oss": passthrough_moe_param_handler}
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def get_moe_param_handler(model_type, ep_enabled):
|
| 175 |
+
"""Resolve the checkpoint export handler for the current MoE layout."""
|
| 176 |
+
if model_type == "gpt_oss" and ep_enabled:
|
| 177 |
+
return streamed_gpt_oss_moe_param_handler
|
| 178 |
+
|
| 179 |
+
return MOE_PARAM_HANDERS.get(model_type, default_moe_param_handler)
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
# ---- EP delta export (veomni-specific converter machinery) ----------------
|
| 183 |
+
# The NaN row probe and the converter entry builder for fused expert params;
|
| 184 |
+
# the DTensor-generic delta pipeline lives in verl.workers.engine.utils.
|
| 185 |
+
|
| 186 |
+
NO_SLOTS_MSG = (
|
| 187 |
+
"converter specs without an enumerable slot table (hf_slots) are not supported by "
|
| 188 |
+
"the sender-side HF delta export; rewrite the converter as a dim-0-separable "
|
| 189 |
+
"to_hf_chunk + hf_slots (see #7060)"
|
| 190 |
+
)
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
_SLOT_TABLES: dict = {}
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def enumerate_hf_slots(process_func, fqn, full_shape, dtype, device=None) -> list:
|
| 197 |
+
"""Let the handler itself reveal its slot table: run it once per expert row
|
| 198 |
+
on a zero dummy row and record the ``(hf_name, hf_shape)`` sequence in the
|
| 199 |
+
handler's own output order. Same philosophy as the mcore probe -- the
|
| 200 |
+
converter owns its output structure, nothing is hand-copied -- so ANY
|
| 201 |
+
handler following the ``(name, segment, expert_id_base)`` contract is
|
| 202 |
+
supported, not just the default one.
|
| 203 |
+
|
| 204 |
+
The table is static metadata (names/shapes depend on the arguments only,
|
| 205 |
+
never on weight values), so it is enumerated once per (handler, fqn, shape,
|
| 206 |
+
dtype) and cached for the process lifetime -- the export calls this every
|
| 207 |
+
sync. Pass the param's device: the handler moves its outputs to the
|
| 208 |
+
accelerator, so an on-device dummy makes that a no-op view instead of a
|
| 209 |
+
per-row H2D copy."""
|
| 210 |
+
n_experts = int(full_shape[0])
|
| 211 |
+
row_shape = tuple(int(dim) for dim in full_shape[1:])
|
| 212 |
+
cache_key = (process_func, fqn, (n_experts, *row_shape), dtype)
|
| 213 |
+
cached = _SLOT_TABLES.get(cache_key)
|
| 214 |
+
if cached is not None:
|
| 215 |
+
return cached
|
| 216 |
+
|
| 217 |
+
dummy_row = torch.zeros((1, *row_shape), dtype=dtype, device=device)
|
| 218 |
+
slots = []
|
| 219 |
+
for expert_id in range(n_experts):
|
| 220 |
+
for hf_name, hf_tensor in process_func(fqn, dummy_row, expert_id_base=expert_id):
|
| 221 |
+
slots.append((hf_name, tuple(hf_tensor.shape)))
|
| 222 |
+
_SLOT_TABLES[cache_key] = slots
|
| 223 |
+
return slots
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
def convert_row_to_hf(name, spec, expert_id: int, pos_in_row, vals, like):
|
| 227 |
+
"""NaN-probe one dim-0 row through the spec's converter: scatter the row's
|
| 228 |
+
(within-row position, value) pairs into a NaN-filled row buffer, run
|
| 229 |
+
``to_hf_chunk`` on it, and extract each output slot's surviving positions and
|
| 230 |
+
values (the converter is a pure permutation, so non-NaN survivors are exactly
|
| 231 |
+
the input pairs in final HF coordinates). ``like`` only lends its
|
| 232 |
+
dtype/device to the buffer. Returns
|
| 233 |
+
``[(slot_in_row, idx_int32, val), ...]``, skipping empty slots.
|
| 234 |
+
|
| 235 |
+
NaN is the sentinel, so a delta whose NEW value is literally NaN is
|
| 236 |
+
indistinguishable from "untouched" and silently dropped -- acceptable
|
| 237 |
+
because NaN weights are already a fatal training failure, and the
|
| 238 |
+
idempotence verify sweep surfaces the divergence on its next pass."""
|
| 239 |
+
row_shape = spec.full_shape[1:]
|
| 240 |
+
row_numel = max(_prodshape(row_shape), 1)
|
| 241 |
+
slots_per_row = len(spec.hf_slots) // int(spec.full_shape[0])
|
| 242 |
+
|
| 243 |
+
row_buf = torch.full((row_numel,), float("nan"), dtype=like.dtype, device=like.device)
|
| 244 |
+
row_buf[pos_in_row] = vals
|
| 245 |
+
hf_outputs = spec.to_hf_chunk(int(expert_id), row_buf.view(1, *row_shape))
|
| 246 |
+
assert len(hf_outputs) == slots_per_row, (
|
| 247 |
+
f"{name}: to_hf_chunk gave {len(hf_outputs)} outputs/row, slot table expects {slots_per_row}"
|
| 248 |
+
)
|
| 249 |
+
|
| 250 |
+
converted = []
|
| 251 |
+
for slot_in_row, (_hf_name, hf_tensor) in enumerate(hf_outputs):
|
| 252 |
+
flat = hf_tensor.reshape(-1)
|
| 253 |
+
if flat.device != like.device:
|
| 254 |
+
# handlers move outputs to the accelerator; under CPUOffloadPolicy the
|
| 255 |
+
# shard (and the rest of this entry) lives on CPU -- normalize so the
|
| 256 |
+
# gather queue never cats mixed-device pieces.
|
| 257 |
+
flat = flat.to(like.device)
|
| 258 |
+
survivor_pos = (~torch.isnan(flat)).nonzero(as_tuple=False).view(-1)
|
| 259 |
+
if survivor_pos.numel():
|
| 260 |
+
converted.append((slot_in_row, survivor_pos.to(torch.int32), flat[survivor_pos]))
|
| 261 |
+
return converted
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
def hf_entry_converter(name, spec, place, lidx, lval):
|
| 265 |
+
"""Turn one converter param's shard-local delta into its final HF-coordinate
|
| 266 |
+
entry ``(slots, dtype_str, counts, idx_concat, val_concat)``: only the touched
|
| 267 |
+
dim-0 rows go through the NaN probe; every rank enumerates the same slot list
|
| 268 |
+
(zero counts when untouched) so the engine's batched gather stays aligned."""
|
| 269 |
+
from verl.workers.engine.spec import translate_flat_indices
|
| 270 |
+
|
| 271 |
+
row_numel = max(_prodshape(spec.full_shape[1:]), 1)
|
| 272 |
+
n_slots = len(spec.hf_slots)
|
| 273 |
+
slots_per_row = n_slots // int(spec.full_shape[0])
|
| 274 |
+
counts = torch.zeros(n_slots, dtype=torch.int64)
|
| 275 |
+
idx_pieces: list = []
|
| 276 |
+
val_pieces: list = []
|
| 277 |
+
if lidx.numel():
|
| 278 |
+
# shard-local flat positions -> full-tensor flat positions, sorted so
|
| 279 |
+
# every touched expert row becomes one contiguous run
|
| 280 |
+
full_idx = translate_flat_indices(lidx, place)
|
| 281 |
+
order = torch.argsort(full_idx)
|
| 282 |
+
full_idx, full_val = full_idx[order], lval[order]
|
| 283 |
+
row_of = torch.div(full_idx, row_numel, rounding_mode="floor")
|
| 284 |
+
touched_rows, run_lengths = torch.unique_consecutive(row_of, return_counts=True)
|
| 285 |
+
cursor = 0
|
| 286 |
+
for expert_id, run_len in zip(touched_rows.tolist(), run_lengths.tolist(), strict=False):
|
| 287 |
+
run_idx = full_idx[cursor : cursor + run_len]
|
| 288 |
+
run_val = full_val[cursor : cursor + run_len]
|
| 289 |
+
cursor += run_len
|
| 290 |
+
pos_in_row = run_idx - expert_id * row_numel
|
| 291 |
+
for slot_in_row, hf_idx, hf_val in convert_row_to_hf(name, spec, expert_id, pos_in_row, run_val, lval):
|
| 292 |
+
counts[int(expert_id) * slots_per_row + slot_in_row] = hf_idx.numel()
|
| 293 |
+
idx_pieces.append(hf_idx)
|
| 294 |
+
val_pieces.append(hf_val)
|
| 295 |
+
if idx_pieces:
|
| 296 |
+
entry_idx = torch.cat(idx_pieces)
|
| 297 |
+
entry_val = torch.cat(val_pieces)
|
| 298 |
+
else:
|
| 299 |
+
entry_idx = torch.empty(0, dtype=torch.int32, device=lval.device)
|
| 300 |
+
entry_val = torch.empty(0, dtype=lval.dtype, device=lval.device)
|
| 301 |
+
return spec.hf_slots, str(lval.dtype).replace("torch.", ""), counts, entry_idx, entry_val
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
def veomni_shard_export(module):
|
| 305 |
+
"""Build the per-rank shard export ``(gen, None)`` for
|
| 306 |
+
:meth:`VeOmniEngine.get_per_tensor_param_shard`.
|
| 307 |
+
|
| 308 |
+
Dense params pass their FSDP DTensor placement through verbatim (flat
|
| 309 |
+
``Shard(0)`` fast path). Grouped expert params are split twice: the expert
|
| 310 |
+
dim (0) is split *manually* across ``ep_group`` (outside DTensor), and the
|
| 311 |
+
local expert block is FSDP-sharded as a DTensor. That combination is not
|
| 312 |
+
expressible as DTensor placements, so the spec carries an explicit
|
| 313 |
+
:class:`BlockPlacement` in a virtual full tensor of all experts
|
| 314 |
+
(dim-0 offset = ``ep_rank * n_local_experts`` + the DTensor block offset)
|
| 315 |
+
with the world group as the gather group, and a ``to_hf_chunk`` closure over
|
| 316 |
+
the veomni MoE param handler (pure slice + rename: fused -> per-expert HF
|
| 317 |
+
names) plus its ``hf_slots`` output enumeration. ``ep_size=1`` is the same
|
| 318 |
+
geometry with the manual split degenerate; Replicate dims on the block mesh
|
| 319 |
+
(newer veomni HSDPs the expert block) dedupe via ``spec.contributes``.
|
| 320 |
+
"""
|
| 321 |
+
import torch.distributed as dist
|
| 322 |
+
from veomni.distributed import parallel_state
|
| 323 |
+
|
| 324 |
+
from verl.utils.model import convert_weight_keys
|
| 325 |
+
|
| 326 |
+
params = module.state_dict()
|
| 327 |
+
params = convert_weight_keys(params, getattr(module, "_fsdp_wrapped_module", module))
|
| 328 |
+
|
| 329 |
+
ps = parallel_state.get_parallel_state()
|
| 330 |
+
model_type = getattr(module.config, "model_type", "default")
|
| 331 |
+
# Delta export must be separable by expert row even when EP is disabled.
|
| 332 |
+
# The packed GPT-OSS handler produces one global tensor and therefore cannot
|
| 333 |
+
# describe sparse row updates; use vLLM's streamed expert format here.
|
| 334 |
+
process_func = (
|
| 335 |
+
streamed_gpt_oss_moe_param_handler
|
| 336 |
+
if model_type == "gpt_oss"
|
| 337 |
+
else get_moe_param_handler(model_type, ps.ep_enabled)
|
| 338 |
+
)
|
| 339 |
+
|
| 340 |
+
from torch.distributed.tensor import DTensor
|
| 341 |
+
from torch.distributed.tensor._utils import compute_local_shape_and_global_offset
|
| 342 |
+
|
| 343 |
+
from ..spec import BlockPlacement, ShardSpec
|
| 344 |
+
|
| 345 |
+
# Structural fact source: ParallelPlan.apply hangs fqn -> SpecInfo on the
|
| 346 |
+
# model itself (it survives state_dict, unlike per-param attributes). Every
|
| 347 |
+
# param gets an entry when EP is on -- non-sliced ones with
|
| 348 |
+
# placement=Replicate() and the SAME para_name -- so the discriminator is
|
| 349 |
+
# the placement type, never ``para_name`` (veomni's own DCP checkpointer
|
| 350 |
+
# keys off ``isinstance(placement, Shard)`` too).
|
| 351 |
+
fqn2si = getattr(module, "_fqn2spec_info", None) or getattr(
|
| 352 |
+
getattr(module, "_fsdp_wrapped_module", module), "_fqn2spec_info", None
|
| 353 |
+
)
|
| 354 |
+
|
| 355 |
+
def _is_ep_param(name, param):
|
| 356 |
+
if fqn2si is not None:
|
| 357 |
+
si = fqn2si.get(name)
|
| 358 |
+
if si is not None:
|
| 359 |
+
from torch.distributed.tensor import Shard
|
| 360 |
+
|
| 361 |
+
return isinstance(si.placement, Shard)
|
| 362 |
+
# fallback: key conversion may have renamed the fqn; historical name match.
|
| 363 |
+
return "mlp.experts." in name and any(p in name for p in ["down_proj", "gate_proj", "up_proj", "gate_up_proj"])
|
| 364 |
+
|
| 365 |
+
def _gen():
|
| 366 |
+
for name, param in params.items():
|
| 367 |
+
# NOT gated on ep_enabled: the fused expert stack exists (and needs
|
| 368 |
+
# the per-expert HF renaming, like the seed path does) even with
|
| 369 |
+
# ep_size=1 -- that is just the degenerate ep_rank=0 geometry.
|
| 370 |
+
is_expert = _is_ep_param(name, param)
|
| 371 |
+
if param.is_floating_point() and param.dtype != torch.bfloat16:
|
| 372 |
+
# mixed-precision keeps fp32 master weights; the wire (and the
|
| 373 |
+
# rollout side) speak bf16, so cast before diffing.
|
| 374 |
+
param = param.to(torch.bfloat16)
|
| 375 |
+
if not is_expert:
|
| 376 |
+
spec = ShardSpec.from_param(param)
|
| 377 |
+
local = param.to_local() if isinstance(param, DTensor) else param
|
| 378 |
+
yield name, local.reshape(-1), spec
|
| 379 |
+
continue
|
| 380 |
+
|
| 381 |
+
# local fused block: [n_local_experts, ...] (ep split already applied)
|
| 382 |
+
contributes = True
|
| 383 |
+
if isinstance(param, DTensor):
|
| 384 |
+
lshape, goff = compute_local_shape_and_global_offset(
|
| 385 |
+
param.shape, param.device_mesh, list(param.placements)
|
| 386 |
+
)
|
| 387 |
+
local = param.to_local()
|
| 388 |
+
# Replicate mesh dims (newer veomni HSDPs the expert block:
|
| 389 |
+
# (ep_replicate, ep_fsdp) mesh) yield identical blocks on every
|
| 390 |
+
# replica -- offsets are already 0 there, so the block math is
|
| 391 |
+
# untouched; only coord-0 ranks contribute, the rest keep
|
| 392 |
+
# lockstep with an empty delta.
|
| 393 |
+
coord = param.device_mesh.get_coordinate()
|
| 394 |
+
rep_dims = [d for d, p in enumerate(param.placements) if p.is_replicate()]
|
| 395 |
+
if coord is not None:
|
| 396 |
+
contributes = all(int(coord[d]) == 0 for d in rep_dims)
|
| 397 |
+
else:
|
| 398 |
+
lshape, goff = tuple(param.shape), (0,) * param.ndim
|
| 399 |
+
local = param
|
| 400 |
+
ep_size = ps.ep_size if ps.ep_enabled else 1
|
| 401 |
+
ep_rank = ps.ep_rank if ps.ep_enabled else 0
|
| 402 |
+
n_local = int(param.shape[0])
|
| 403 |
+
full_shape = (n_local * ep_size,) + tuple(param.shape[1:])
|
| 404 |
+
offset = (ep_rank * n_local + int(goff[0]),) + tuple(int(o) for o in goff[1:])
|
| 405 |
+
block = BlockPlacement(tuple(int(x) for x in lshape), offset, full_shape)
|
| 406 |
+
|
| 407 |
+
# the blocks of one expert tensor are spread over ep x (block mesh,
|
| 408 |
+
# replicas included); v1 requires that to cover the whole trainer
|
| 409 |
+
# world so the world group doubles as the gather group (rank 0 ==
|
| 410 |
+
# engine master). Replicas are deduplicated via ``contributes``.
|
| 411 |
+
mesh_world = param.device_mesh.size() if isinstance(param, DTensor) else 1
|
| 412 |
+
assert ep_size * mesh_world == dist.get_world_size(), (
|
| 413 |
+
f"expert blocks must cover the trainer world: ep={ep_size} x "
|
| 414 |
+
f"block mesh={mesh_world} != world={dist.get_world_size()}"
|
| 415 |
+
)
|
| 416 |
+
spec = ShardSpec(
|
| 417 |
+
full_shape=full_shape,
|
| 418 |
+
place=block,
|
| 419 |
+
gather_group=dist.group.WORLD,
|
| 420 |
+
contributes=contributes,
|
| 421 |
+
# dim-0-separable: a segment is a run of whole experts starting at
|
| 422 |
+
# global expert id ``start`` -- exactly the handler's contract.
|
| 423 |
+
to_hf_chunk=lambda start, seg, _f=process_func, _n=name: list(_f(_n, seg, expert_id_base=start)),
|
| 424 |
+
# slot table enables sender-side conversion; probed once from
|
| 425 |
+
# the handler itself, so any handler is supported.
|
| 426 |
+
hf_slots=enumerate_hf_slots(process_func, name, full_shape, torch.bfloat16, device=local.device),
|
| 427 |
+
)
|
| 428 |
+
yield name, local.reshape(-1), spec
|
| 429 |
+
|
| 430 |
+
return _gen(), None
|
repro/vendor/verl/workers/rollout/naive/__init__.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2024 Bytedance Ltd. and/or its affiliates
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
|
| 15 |
+
from .naive_rollout import NaiveRollout
|
| 16 |
+
|
| 17 |
+
__all__ = ["NaiveRollout"]
|
repro/vendor/verl/workers/rollout/naive/naive_rollout.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2024 Bytedance Ltd. and/or its affiliates
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""
|
| 15 |
+
In single GPU rollout, the sequences are generated directly by sampling from the model.
|
| 16 |
+
The output will contain
|
| 17 |
+
1. output_ids
|
| 18 |
+
2. attention_masks (left padding)
|
| 19 |
+
3. eos_masks
|
| 20 |
+
4. log_probs
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
import torch
|
| 24 |
+
import torch.nn.functional as F
|
| 25 |
+
from tensordict import TensorDict
|
| 26 |
+
from torch import nn
|
| 27 |
+
|
| 28 |
+
from verl import DataProto
|
| 29 |
+
from verl.utils.torch_functional import logprobs_from_logits
|
| 30 |
+
|
| 31 |
+
from ..base import BaseRollout
|
| 32 |
+
|
| 33 |
+
__all__ = ["NaiveRollout"]
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class NaiveRollout(BaseRollout):
|
| 37 |
+
def __init__(self, module: nn.Module, config):
|
| 38 |
+
"""A naive rollout. It requires the module to be compatible with huggingface APIs. That is:
|
| 39 |
+
The module should define __call__ to receive input_ids, attention_mask and position_ids.
|
| 40 |
+
It outputs a structure that contains logits field.
|
| 41 |
+
|
| 42 |
+
Args:
|
| 43 |
+
module: module here follows huggingface APIs
|
| 44 |
+
config: DictConfig
|
| 45 |
+
"""
|
| 46 |
+
super().__init__()
|
| 47 |
+
self.config = config
|
| 48 |
+
self.module = module
|
| 49 |
+
|
| 50 |
+
@torch.no_grad()
|
| 51 |
+
def generate_sequences(self, prompts: DataProto) -> DataProto:
|
| 52 |
+
"""Generate sequences"""
|
| 53 |
+
idx = prompts.batch["input_ids"] # (bs, prompt_length)
|
| 54 |
+
attention_mask = prompts.batch["attention_mask"] # left-padded attention_mask
|
| 55 |
+
position_ids = prompts.batch["position_ids"]
|
| 56 |
+
|
| 57 |
+
# used to construct attention_mask
|
| 58 |
+
eos_token_id = prompts.meta_info["eos_token_id"]
|
| 59 |
+
|
| 60 |
+
batch_size = idx.size(0)
|
| 61 |
+
prompt_length = idx.size(1)
|
| 62 |
+
|
| 63 |
+
self.module.eval()
|
| 64 |
+
|
| 65 |
+
prev_attention_mask = torch.ones(size=(batch_size, 1), dtype=attention_mask.dtype, device=attention_mask.device)
|
| 66 |
+
|
| 67 |
+
logits_lst = []
|
| 68 |
+
for _ in range(self.config.response_length):
|
| 69 |
+
# if the sequence context is growing too long we must crop it at block_size
|
| 70 |
+
# idx_cond = idx if idx.size(1) <= self.config.block_size else idx[:, -self.config.block_size:]
|
| 71 |
+
idx_cond = idx
|
| 72 |
+
# forward the model to get the logits for the index in the sequence
|
| 73 |
+
# we use huggingface APIs here
|
| 74 |
+
output = self.module(input_ids=idx_cond, attention_mask=attention_mask, position_ids=position_ids)
|
| 75 |
+
logits = output.logits
|
| 76 |
+
# pluck the logits at the final step and scale by desired temperature
|
| 77 |
+
logits = logits[:, -1, :] / self.config.temperature # (bs, vocab_size)
|
| 78 |
+
# optionally crop the logits to only the top k options
|
| 79 |
+
if self.config.top_k is not None:
|
| 80 |
+
v, _ = torch.topk(logits, min(self.config.top_k, logits.size(-1)))
|
| 81 |
+
logits[logits < v[:, [-1]]] = -float("Inf")
|
| 82 |
+
# apply softmax to convert logits to (normalized) probabilities
|
| 83 |
+
probs = F.softmax(logits, dim=-1)
|
| 84 |
+
# sample from the distribution
|
| 85 |
+
if self.config.do_sample:
|
| 86 |
+
idx_next = torch.multinomial(probs, num_samples=1)
|
| 87 |
+
else:
|
| 88 |
+
idx_next = torch.argmax(probs, dim=-1, keepdim=True)
|
| 89 |
+
|
| 90 |
+
attention_mask = torch.cat((attention_mask, prev_attention_mask), dim=-1)
|
| 91 |
+
|
| 92 |
+
for token_id in eos_token_id:
|
| 93 |
+
prev_attention_mask = torch.logical_and(idx_next != token_id, prev_attention_mask.bool())
|
| 94 |
+
prev_attention_mask.to(attention_mask.dtype)
|
| 95 |
+
|
| 96 |
+
position_ids = torch.cat((position_ids, position_ids[:, -1:] + 1), dim=-1)
|
| 97 |
+
|
| 98 |
+
# append sampled index to the running sequence and continue
|
| 99 |
+
idx = torch.cat((idx, idx_next), dim=1)
|
| 100 |
+
logits_lst.append(logits)
|
| 101 |
+
|
| 102 |
+
logits = torch.stack(logits_lst, dim=1) # (bs, response_length, vocab_size)
|
| 103 |
+
prompts = idx[:, :prompt_length] # (bs, prompt_length)
|
| 104 |
+
response = idx[:, prompt_length:] # (bs, response_length)
|
| 105 |
+
log_probs = logprobs_from_logits(logits=logits, labels=response)
|
| 106 |
+
batch = TensorDict(
|
| 107 |
+
{
|
| 108 |
+
"input_ids": prompts,
|
| 109 |
+
"responses": response,
|
| 110 |
+
"sequences": idx,
|
| 111 |
+
"old_log_probs": log_probs,
|
| 112 |
+
"attention_mask": attention_mask,
|
| 113 |
+
"position_ids": position_ids,
|
| 114 |
+
},
|
| 115 |
+
batch_size=batch_size,
|
| 116 |
+
)
|
| 117 |
+
|
| 118 |
+
self.module.train()
|
| 119 |
+
|
| 120 |
+
return DataProto(batch=batch)
|
repro/vendor/verl/workers/rollout/sglang_rollout/__init__.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2024 Bytedance Ltd. and/or its affiliates
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
repro/vendor/verl/workers/rollout/sglang_rollout/delta_loader.py
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025 Bytedance Ltd. and/or its affiliates
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""In-process sparse delta apply for SGLang, loaded via ``--custom-weight-loader``.
|
| 15 |
+
|
| 16 |
+
SGLang's ``update_weights_from_tensor`` supports pluggable loaders: when the
|
| 17 |
+
request's ``load_format`` names an import path registered in
|
| 18 |
+
``--custom-weight-loader``, SGLang ``dynamic_import``s it **inside every TP
|
| 19 |
+
worker process** and calls ``loader(model, named_tensors)``. That is exactly
|
| 20 |
+
the hook a sparse delta needs: the delta payload is decoded and applied *in
|
| 21 |
+
place* onto SGLang's live weights (masked overwrite of only the changed
|
| 22 |
+
positions), so the receiver never stages a full-model mirror anywhere —
|
| 23 |
+
peak memory is one decode chunk, independent of model size — and no SGLang
|
| 24 |
+
fork or patch is required.
|
| 25 |
+
|
| 26 |
+
Wire contract (what the delta checkpoint engine sends per flush):
|
| 27 |
+
|
| 28 |
+
* ``__delta_spec__`` — uint8 tensor holding a JSON manifest
|
| 29 |
+
``{"encoding", "params": [DeltaParam-dict...], "checksum"}``.
|
| 30 |
+
* ``__positions__`` — uint8 blob of packed positions (per-param slices are
|
| 31 |
+
byte offsets ``pos_start:pos_end``; ``indices`` packs little-endian int32
|
| 32 |
+
absolute positions).
|
| 33 |
+
* ``__values__`` — the changed values in the flush's (uniform) dtype;
|
| 34 |
+
per-param slices are element offsets ``val_start:val_end``.
|
| 35 |
+
|
| 36 |
+
Register at server launch (verl config)::
|
| 37 |
+
|
| 38 |
+
+actor_rollout_ref.rollout.engine_kwargs.sglang.custom_weight_loader='["verl.workers.rollout.sglang_rollout.delta_loader.apply_delta"]'
|
| 39 |
+
"""
|
| 40 |
+
|
| 41 |
+
from __future__ import annotations
|
| 42 |
+
|
| 43 |
+
import json
|
| 44 |
+
import logging
|
| 45 |
+
import math
|
| 46 |
+
from collections.abc import Iterable, Iterator
|
| 47 |
+
from contextlib import contextmanager
|
| 48 |
+
|
| 49 |
+
import torch
|
| 50 |
+
|
| 51 |
+
logger = logging.getLogger(__name__)
|
| 52 |
+
|
| 53 |
+
# Cap on the densified tensors handed to one model.load_weights call, matching
|
| 54 |
+
# SGLang's own delta-apply chunking default.
|
| 55 |
+
CHUNK_BYTES = 512 << 20
|
| 56 |
+
|
| 57 |
+
# Import path callers pass as both --custom-weight-loader and load_format.
|
| 58 |
+
LOADER_FQN = "verl.workers.rollout.sglang_rollout.delta_loader.apply_delta"
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def apply_delta(model: torch.nn.Module, named_tensors: Iterable[tuple[str, torch.Tensor]]) -> None:
|
| 62 |
+
"""Decode one sparse delta flush and masked-apply it onto ``model`` in place."""
|
| 63 |
+
from verl.checkpoint_engine.delta_sync.encode import checksum as _checksum
|
| 64 |
+
|
| 65 |
+
tensors = dict(named_tensors)
|
| 66 |
+
spec = json.loads(bytes(tensors["__delta_spec__"].cpu().numpy().tobytes()).decode())
|
| 67 |
+
values = tensors["__values__"]
|
| 68 |
+
positions = tensors.get("__positions__")
|
| 69 |
+
if positions is None: # values-only flush (the seed) carries no positions
|
| 70 |
+
positions = torch.empty(0, dtype=torch.uint8, device=values.device)
|
| 71 |
+
|
| 72 |
+
got = _checksum(positions, values)
|
| 73 |
+
if got != int(spec["checksum"]):
|
| 74 |
+
raise RuntimeError(
|
| 75 |
+
f"delta checksum mismatch in sglang loader: got {got}, expected {spec['checksum']}; "
|
| 76 |
+
"indicates corruption between sender encode and receiver apply"
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
if spec["encoding"] == "dense":
|
| 80 |
+
if spec.get("verify"):
|
| 81 |
+
_verify_dense(model, spec["params"], values, bool(spec.get("is_last")))
|
| 82 |
+
return
|
| 83 |
+
_apply_dense(model, spec["params"], values)
|
| 84 |
+
return
|
| 85 |
+
|
| 86 |
+
encoding = spec["encoding"]
|
| 87 |
+
with _masked_copy():
|
| 88 |
+
chunk: list[tuple[str, torch.Tensor]] = []
|
| 89 |
+
chunk_bytes = 0
|
| 90 |
+
for p in spec["params"]:
|
| 91 |
+
t = _decode_one(encoding, positions, values, p)
|
| 92 |
+
nbytes = t.numel() * t.element_size()
|
| 93 |
+
if chunk and chunk_bytes + nbytes > CHUNK_BYTES:
|
| 94 |
+
model.load_weights(chunk)
|
| 95 |
+
chunk, chunk_bytes = [], 0
|
| 96 |
+
chunk.append((p["name"], t))
|
| 97 |
+
chunk_bytes += nbytes
|
| 98 |
+
if chunk:
|
| 99 |
+
model.load_weights(chunk)
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def _apply_dense(model: torch.nn.Module, params: list[dict], values: torch.Tensor) -> None:
|
| 103 |
+
"""Apply a dense (full-coverage) flush: plain chunked load, no masking needed."""
|
| 104 |
+
chunk: list[tuple[str, torch.Tensor]] = []
|
| 105 |
+
chunk_bytes = 0
|
| 106 |
+
for p in params:
|
| 107 |
+
dtype = getattr(torch, p["dtype"])
|
| 108 |
+
t = values[p["val_start"] : p["val_end"]].to(dtype).view(p["shape"])
|
| 109 |
+
nbytes = t.numel() * t.element_size()
|
| 110 |
+
if chunk and chunk_bytes + nbytes > CHUNK_BYTES:
|
| 111 |
+
model.load_weights(chunk)
|
| 112 |
+
chunk, chunk_bytes = [], 0
|
| 113 |
+
chunk.append((p["name"], t))
|
| 114 |
+
chunk_bytes += nbytes
|
| 115 |
+
if chunk:
|
| 116 |
+
model.load_weights(chunk)
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
_VERIFY_STATS: dict = {"params": 0, "pieces": []}
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def _verify_dense(model: torch.nn.Module, params: list[dict], values: torch.Tensor, is_last: bool) -> None:
|
| 123 |
+
"""State-equivalence sweep, phrased as an IDEMPOTENCE check: replaying the
|
| 124 |
+
trainer's FULL current weights onto an in-sync server must be a no-op. For
|
| 125 |
+
each parameter we snapshot every ``copy_`` destination (the exact internal
|
| 126 |
+
slices sglang's own ``load_weights`` name-mapping resolves) BEFORE the
|
| 127 |
+
load, run the real load path -- including multi-stage loaders that first
|
| 128 |
+
write raw values and then transform in place (e.g. mamba's
|
| 129 |
+
``A = -exp(A_log)`` composed loader) -- and bit-compare the post-load state
|
| 130 |
+
against the snapshot. Any changed element means the server's
|
| 131 |
+
delta-accumulated state disagreed with the trainer's; that fails loud.
|
| 132 |
+
Comparing per copy_ call instead would false-positive on every
|
| 133 |
+
transform-loaded param (raw-vs-transformed at each stage)."""
|
| 134 |
+
orig_copy = torch.Tensor.copy_
|
| 135 |
+
|
| 136 |
+
by_param = _VERIFY_STATS.setdefault("by_param", {})
|
| 137 |
+
for p in params:
|
| 138 |
+
touched: dict = {}
|
| 139 |
+
|
| 140 |
+
def snap_then_copy_(self: torch.Tensor, src: torch.Tensor, *args, _touched=touched, **kwargs) -> torch.Tensor:
|
| 141 |
+
key = (self.data_ptr(), self.numel(), self.dtype)
|
| 142 |
+
if key not in _touched:
|
| 143 |
+
_touched[key] = (self, self.detach().clone())
|
| 144 |
+
return orig_copy(self, src, *args, **kwargs)
|
| 145 |
+
|
| 146 |
+
torch.Tensor.copy_ = snap_then_copy_
|
| 147 |
+
try:
|
| 148 |
+
_apply_dense(model, [p], values)
|
| 149 |
+
finally:
|
| 150 |
+
torch.Tensor.copy_ = orig_copy
|
| 151 |
+
bad = 0
|
| 152 |
+
for dst, pre in touched.values():
|
| 153 |
+
if dst.is_floating_point() and dst.element_size() == 2:
|
| 154 |
+
bad += int((dst.view(torch.int16) != pre.view(torch.int16)).sum())
|
| 155 |
+
else:
|
| 156 |
+
bad += int((dst != pre).sum())
|
| 157 |
+
if bad:
|
| 158 |
+
by_param[p["name"]] = by_param.get(p["name"], 0) + bad
|
| 159 |
+
_VERIFY_STATS["pieces"].append(bad)
|
| 160 |
+
_VERIFY_STATS["params"] += len(params)
|
| 161 |
+
if is_last:
|
| 162 |
+
total = sum(_VERIFY_STATS["pieces"])
|
| 163 |
+
n = _VERIFY_STATS["params"]
|
| 164 |
+
_VERIFY_STATS["params"] = 0
|
| 165 |
+
_VERIFY_STATS["pieces"] = []
|
| 166 |
+
offenders = _VERIFY_STATS.pop("by_param", {})
|
| 167 |
+
top = sorted(offenders.items(), key=lambda kv: -kv[1])[:12]
|
| 168 |
+
logger.warning("DELTA-VERIFY sweep: params=%d mismatch_elems=%d offenders=%s", n, total, top)
|
| 169 |
+
if total:
|
| 170 |
+
raise RuntimeError(
|
| 171 |
+
f"delta state verification FAILED: {total} elements differ between the "
|
| 172 |
+
f"server's delta-accumulated weights and the trainer's full export; "
|
| 173 |
+
f"top offenders: {top}"
|
| 174 |
+
)
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def _decode_one(encoding: str, positions: torch.Tensor, values: torch.Tensor, p: dict) -> torch.Tensor:
|
| 178 |
+
"""Densify one param's sparse delta into a full-shape NaN-masked tensor.
|
| 179 |
+
|
| 180 |
+
``indices`` positions are reinterpreted via an int32 view (8 B/element
|
| 181 |
+
transient) rather than a per-byte int64 unpack (32 B/element), so even a
|
| 182 |
+
full-seed flush of a large embedding stays within a few GiB.
|
| 183 |
+
"""
|
| 184 |
+
numel = math.prod(p["shape"])
|
| 185 |
+
dtype = getattr(torch, p["dtype"])
|
| 186 |
+
flat = torch.full((numel,), float("nan"), dtype=dtype, device=values.device)
|
| 187 |
+
vals = values[p["val_start"] : p["val_end"]]
|
| 188 |
+
if vals.numel() == 0:
|
| 189 |
+
return flat.view(p["shape"])
|
| 190 |
+
|
| 191 |
+
pos_b = positions[p["pos_start"] : p["pos_end"]]
|
| 192 |
+
if encoding == "indices":
|
| 193 |
+
# pos_start is always int32-aligned (each entry packs nnz * 4 bytes).
|
| 194 |
+
idx = pos_b.view(torch.int32).to(torch.int64)
|
| 195 |
+
else:
|
| 196 |
+
raise ValueError(f"unsupported delta encoding: {encoding!r}")
|
| 197 |
+
|
| 198 |
+
flat.index_copy_(0, idx, vals.to(dtype))
|
| 199 |
+
return flat.view(p["shape"])
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
@contextmanager
|
| 203 |
+
def _masked_copy() -> Iterator[None]:
|
| 204 |
+
"""Temporarily make ``Tensor.copy_`` skip NaN positions in the source.
|
| 205 |
+
|
| 206 |
+
SGLang's per-model ``load_weights`` ultimately lands on ``param.copy_(loaded)``
|
| 207 |
+
(possibly on a narrowed TP slice). Under this context a NaN-masked source
|
| 208 |
+
overwrites only the changed positions; fully dense sources (e.g. the first
|
| 209 |
+
full-seed flush) take the original fast path untouched.
|
| 210 |
+
"""
|
| 211 |
+
orig_copy = torch.Tensor.copy_
|
| 212 |
+
|
| 213 |
+
def masked_copy_(self: torch.Tensor, src: torch.Tensor, *args, **kwargs) -> torch.Tensor:
|
| 214 |
+
# Sync-free masked overwrite: boolean advanced indexing (and a
|
| 215 |
+
# ``mask.all()`` early-out) would force a device->host sync per
|
| 216 |
+
# parameter -- ruinous for MoE flushes carrying >10k per-expert
|
| 217 |
+
# entries. ``torch.where`` keeps everything on-stream; a NaN-free
|
| 218 |
+
# (dense) source degenerates to a plain copy.
|
| 219 |
+
if isinstance(src, torch.Tensor) and src.is_floating_point() and self.shape == src.shape:
|
| 220 |
+
cast = src.to(self.dtype)
|
| 221 |
+
return orig_copy(self, torch.where(torch.isnan(cast), self, cast))
|
| 222 |
+
return orig_copy(self, src, *args, **kwargs)
|
| 223 |
+
|
| 224 |
+
torch.Tensor.copy_ = masked_copy_
|
| 225 |
+
try:
|
| 226 |
+
yield
|
| 227 |
+
finally:
|
| 228 |
+
torch.Tensor.copy_ = orig_copy
|
repro/vendor/verl/workers/rollout/sglang_rollout/http_server_engine.py
ADDED
|
@@ -0,0 +1,990 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025 z.ai
|
| 2 |
+
# Copyright 2023-2024 SGLang Team
|
| 3 |
+
# Copyright 2025 ModelBest Inc. and/or its affiliates
|
| 4 |
+
#
|
| 5 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 6 |
+
# you may not use this file except in compliance with the License.
|
| 7 |
+
# You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 12 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 13 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 14 |
+
# See the License for the specific language governing permissions and
|
| 15 |
+
# limitations under the License.
|
| 16 |
+
#
|
| 17 |
+
# This file is adapted from multiple sources:
|
| 18 |
+
# 1. THUDM/slime project
|
| 19 |
+
# Original source: https://github.com/THUDM/slime/blob/main/slime/backends/sglang_utils/http_server_engine.py
|
| 20 |
+
# Copyright 2025 z.ai
|
| 21 |
+
# Licensed under the Apache License, Version 2.0
|
| 22 |
+
# 2. SGLang project
|
| 23 |
+
# Original source: https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/entrypoints/http_server_engine.py
|
| 24 |
+
# Copyright 2023-2024 SGLang Team
|
| 25 |
+
# Licensed under the Apache License, Version 2.0
|
| 26 |
+
#
|
| 27 |
+
# Modifications made by z.ai and ModelBest Inc. include but are not limited to:
|
| 28 |
+
# - Enhanced error handling and retry logic
|
| 29 |
+
# - Added async support with connection pooling
|
| 30 |
+
# - Extended functionality for distributed weight updates
|
| 31 |
+
# - Improved logging and monitoring capabilities
|
| 32 |
+
# - Additional configuration options and optimizations
|
| 33 |
+
|
| 34 |
+
"""HTTP Server Engine Adapter for SGLang.
|
| 35 |
+
|
| 36 |
+
This module provides HTTP-based adapters for SGLang engines, allowing communication
|
| 37 |
+
with SGLang servers through HTTP requests instead of direct engine calls.
|
| 38 |
+
|
| 39 |
+
Classes:
|
| 40 |
+
HttpServerAdapter: Synchronous HTTP adapter for SGLang engines
|
| 41 |
+
AsyncHttpServerAdapter: Asynchronous HTTP adapter for SGLang engines
|
| 42 |
+
|
| 43 |
+
Functions:
|
| 44 |
+
launch_server_process: Launch and initialize an SGLang HTTP server process
|
| 45 |
+
"""
|
| 46 |
+
|
| 47 |
+
import asyncio
|
| 48 |
+
import logging
|
| 49 |
+
import multiprocessing
|
| 50 |
+
import os
|
| 51 |
+
import time
|
| 52 |
+
from contextlib import asynccontextmanager
|
| 53 |
+
from typing import Any, Callable, Optional
|
| 54 |
+
|
| 55 |
+
import aiohttp
|
| 56 |
+
import requests
|
| 57 |
+
from sglang.srt.entrypoints.EngineBase import EngineBase
|
| 58 |
+
from sglang.srt.entrypoints.http_server import launch_server
|
| 59 |
+
from sglang.srt.managers.io_struct import UpdateWeightsFromTensorReqInput
|
| 60 |
+
from sglang.srt.server_args import ServerArgs
|
| 61 |
+
from sglang.srt.utils import kill_process_tree
|
| 62 |
+
|
| 63 |
+
# Configure logger
|
| 64 |
+
logger = logging.getLogger(__name__)
|
| 65 |
+
logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
|
| 66 |
+
|
| 67 |
+
# Default configuration constants
|
| 68 |
+
DEFAULT_TIMEOUT = 60.0
|
| 69 |
+
DEFAULT_MAX_ATTEMPTS = 3
|
| 70 |
+
DEFAULT_RETRY_DELAY = 2.0
|
| 71 |
+
DEFAULT_MAX_CONNECTIONS = 2000
|
| 72 |
+
DEFAULT_MAX_WAIT_TIME = 300.0
|
| 73 |
+
# Cap on how much of a failed response body reaches the log.
|
| 74 |
+
_ERROR_BODY_CHARS = 2000
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def _read_response(response: requests.Response):
|
| 78 |
+
if response.status_code == 204 or not response.content:
|
| 79 |
+
return {}
|
| 80 |
+
try:
|
| 81 |
+
return response.json()
|
| 82 |
+
except ValueError:
|
| 83 |
+
return {
|
| 84 |
+
"content_type": response.headers.get("Content-Type", ""),
|
| 85 |
+
"text": response.text,
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def _log_error_body(endpoint: str, status: int, body: str) -> None:
|
| 90 |
+
"""Log a failed response's body; the raised error carries only the status line."""
|
| 91 |
+
logger.error(f"HTTP error for {endpoint}: {status}, body={body[:_ERROR_BODY_CHARS]}")
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
async def _read_async_response(resp: aiohttp.ClientResponse) -> dict[str, Any]:
|
| 95 |
+
if resp.status == 204 or (resp.content_length == 0):
|
| 96 |
+
return {}
|
| 97 |
+
|
| 98 |
+
try:
|
| 99 |
+
return await resp.json(content_type=None)
|
| 100 |
+
except Exception:
|
| 101 |
+
try:
|
| 102 |
+
text = await resp.text()
|
| 103 |
+
except Exception:
|
| 104 |
+
return {}
|
| 105 |
+
return {
|
| 106 |
+
"content_type": (resp.headers.get("Content-Type") or ""),
|
| 107 |
+
"text": text,
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def launch_server_process(
|
| 112 |
+
server_args: ServerArgs,
|
| 113 |
+
timeout: float = DEFAULT_TIMEOUT,
|
| 114 |
+
max_wait_time=DEFAULT_MAX_WAIT_TIME,
|
| 115 |
+
first_rank_in_node=False,
|
| 116 |
+
) -> multiprocessing.Process:
|
| 117 |
+
"""Launch an SGLang HTTP server process and wait for it to be ready.
|
| 118 |
+
|
| 119 |
+
This function starts a new process running an SGLang HTTP server, then waits
|
| 120 |
+
for the server to become ready by polling its health endpoints. It ensures
|
| 121 |
+
the server is fully operational before returning.
|
| 122 |
+
|
| 123 |
+
Args:
|
| 124 |
+
server_args (ServerArgs): Server configuration arguments including host, port, and other settings
|
| 125 |
+
timeout (float, optional): Timeout for individual HTTP requests during health checks.
|
| 126 |
+
Defaults to DEFAULT_TIMEOUT.
|
| 127 |
+
|
| 128 |
+
Returns:
|
| 129 |
+
multiprocessing.Process: The launched multiprocessing.Process instance
|
| 130 |
+
|
| 131 |
+
Raises:
|
| 132 |
+
RuntimeError: If the server process terminates unexpectedly during startup or cache flush
|
| 133 |
+
TimeoutError: If server fails to become ready within reasonable time (300 seconds)
|
| 134 |
+
requests.RequestException: If health check requests fail repeatedly
|
| 135 |
+
|
| 136 |
+
Note:
|
| 137 |
+
This function will return immediately for non-master nodes (node_rank != 0),
|
| 138 |
+
but the process will still be started and returned.
|
| 139 |
+
This is for consistency; except for the process obtained by node_rank = 0,
|
| 140 |
+
other processes have no actual effect.
|
| 141 |
+
"""
|
| 142 |
+
p = multiprocessing.Process(target=launch_server, args=(server_args,))
|
| 143 |
+
if server_args.node_rank != 0 or not first_rank_in_node:
|
| 144 |
+
logger.info(f"Server process started with PID {p.pid} for node rank {server_args.node_rank}", flush=True)
|
| 145 |
+
return p
|
| 146 |
+
|
| 147 |
+
p.start()
|
| 148 |
+
|
| 149 |
+
base_url = server_args.url()
|
| 150 |
+
headers = {
|
| 151 |
+
"Content-Type": "application/json; charset=utf-8",
|
| 152 |
+
"Authorization": f"Bearer {server_args.api_key}",
|
| 153 |
+
}
|
| 154 |
+
|
| 155 |
+
# Health check with overall timeout
|
| 156 |
+
start_time = time.time()
|
| 157 |
+
|
| 158 |
+
with requests.Session() as session:
|
| 159 |
+
while time.time() - start_time < max_wait_time:
|
| 160 |
+
if not p.is_alive():
|
| 161 |
+
raise RuntimeError("Server process terminated unexpectedly during startup")
|
| 162 |
+
|
| 163 |
+
try:
|
| 164 |
+
if server_args.is_embedding:
|
| 165 |
+
response = session.get(f"{base_url}/health", headers=headers, timeout=timeout)
|
| 166 |
+
else:
|
| 167 |
+
response = session.get(f"{base_url}/health_generate", headers=headers, timeout=timeout)
|
| 168 |
+
if response.status_code == 200:
|
| 169 |
+
break
|
| 170 |
+
except requests.RequestException as e:
|
| 171 |
+
logger.debug(f"Health check failed: {e}")
|
| 172 |
+
|
| 173 |
+
time.sleep(2)
|
| 174 |
+
else:
|
| 175 |
+
p.terminate()
|
| 176 |
+
logger.error(f"Server in {base_url} failed to become healthy within timeout period")
|
| 177 |
+
raise TimeoutError("Server failed to become healthy within timeout period")
|
| 178 |
+
|
| 179 |
+
# Ensure cache is ready
|
| 180 |
+
while time.time() - start_time < max_wait_time:
|
| 181 |
+
if not p.is_alive():
|
| 182 |
+
raise RuntimeError("Server process terminated unexpectedly during cache flush")
|
| 183 |
+
|
| 184 |
+
try:
|
| 185 |
+
response = session.get(f"{base_url}/flush_cache", headers=headers, timeout=timeout)
|
| 186 |
+
if response.status_code == 200:
|
| 187 |
+
break
|
| 188 |
+
except requests.RequestException as e:
|
| 189 |
+
logger.debug(f"Cache flush check failed: {e}")
|
| 190 |
+
|
| 191 |
+
time.sleep(2)
|
| 192 |
+
else:
|
| 193 |
+
p.terminate()
|
| 194 |
+
raise TimeoutError("Server cache flush failed within timeout period")
|
| 195 |
+
|
| 196 |
+
return p
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
class HttpServerAdapter(EngineBase):
|
| 200 |
+
"""HTTP-based adapter for SGLang engines.
|
| 201 |
+
|
| 202 |
+
This adapter allows interaction with SGLang engines through HTTP requests
|
| 203 |
+
instead of direct engine calls. It launches an HTTP server process and
|
| 204 |
+
provides methods to communicate with it via REST API calls.
|
| 205 |
+
|
| 206 |
+
You can use this class to launch a server from a HttpServerAdapter instance.
|
| 207 |
+
We recommend using this class only when you need to use http server.
|
| 208 |
+
Otherwise, you can use Engine directly.
|
| 209 |
+
|
| 210 |
+
Attributes:
|
| 211 |
+
router_ip (Optional[str]): IP address of the router for worker registration
|
| 212 |
+
router_port (Optional[int]): Port of the router for worker registration
|
| 213 |
+
server_args (ServerArgs): Server configuration arguments
|
| 214 |
+
node_rank (int): Rank of this node in distributed setup
|
| 215 |
+
process (multiprocessing.Process): The launched server process
|
| 216 |
+
timeout (float): HTTP request timeout in seconds
|
| 217 |
+
max_attempts (int): Maximum number of attempts for requests
|
| 218 |
+
retry_delay (float): Base delay between retries in seconds
|
| 219 |
+
"""
|
| 220 |
+
|
| 221 |
+
def __init__(
|
| 222 |
+
self,
|
| 223 |
+
router_ip: Optional[str] = None,
|
| 224 |
+
router_port: Optional[int] = None,
|
| 225 |
+
timeout: float = DEFAULT_TIMEOUT,
|
| 226 |
+
max_attempts: int = DEFAULT_MAX_ATTEMPTS,
|
| 227 |
+
retry_delay: float = DEFAULT_RETRY_DELAY,
|
| 228 |
+
first_rank_in_node: bool = False,
|
| 229 |
+
max_start_wait_time: float = DEFAULT_MAX_WAIT_TIME,
|
| 230 |
+
launch_server: bool = True,
|
| 231 |
+
**kwargs: Any,
|
| 232 |
+
) -> None:
|
| 233 |
+
"""Initialize the HTTP server engine adapter.
|
| 234 |
+
|
| 235 |
+
Args:
|
| 236 |
+
router_ip (Optional[str], optional): IP address of router for worker registration.
|
| 237 |
+
Defaults to None.
|
| 238 |
+
router_port (Optional[int], optional): Port of router for worker registration.
|
| 239 |
+
Defaults to None.
|
| 240 |
+
timeout (float, optional): HTTP request timeout in seconds.
|
| 241 |
+
Defaults to DEFAULT_TIMEOUT.
|
| 242 |
+
max_attempts (int, optional): Maximum number of retry attempts for failed requests.
|
| 243 |
+
Defaults to DEFAULT_MAX_ATTEMPTS.
|
| 244 |
+
retry_delay (float, optional): Base delay between retries in seconds.
|
| 245 |
+
Defaults to DEFAULT_RETRY_DELAY.
|
| 246 |
+
launch_server (bool, optional): Whether to launch the server process.
|
| 247 |
+
Defaults to True.
|
| 248 |
+
**kwargs (Any): Additional arguments passed to ServerArgs
|
| 249 |
+
|
| 250 |
+
Note:
|
| 251 |
+
TODO: @ChangyiYang Enable SGLang router for this http server engine
|
| 252 |
+
If both router_ip and router_port are provided and this is the master node
|
| 253 |
+
(node_rank == 0), the adapter will automatically register with the router.
|
| 254 |
+
"""
|
| 255 |
+
self.router_ip: Optional[str] = router_ip
|
| 256 |
+
self.router_port: Optional[int] = router_port
|
| 257 |
+
self.timeout: float = timeout
|
| 258 |
+
self.max_attempts: int = max_attempts
|
| 259 |
+
self.retry_delay: float = retry_delay
|
| 260 |
+
self.server_args: ServerArgs = ServerArgs(**kwargs)
|
| 261 |
+
self.node_rank: int = self.server_args.node_rank
|
| 262 |
+
self.max_start_wait_time: float = max_start_wait_time
|
| 263 |
+
|
| 264 |
+
logger.info(
|
| 265 |
+
f"Launch HttpServerAdapter at: {self.server_args.host}:{self.server_args.port} with {first_rank_in_node}"
|
| 266 |
+
)
|
| 267 |
+
if launch_server:
|
| 268 |
+
self.process: multiprocessing.Process = launch_server_process(
|
| 269 |
+
self.server_args, self.timeout, self.max_start_wait_time, first_rank_in_node
|
| 270 |
+
)
|
| 271 |
+
|
| 272 |
+
if self.node_rank == 0 and self.router_ip and self.router_port:
|
| 273 |
+
self._register_with_router()
|
| 274 |
+
|
| 275 |
+
def _register_with_router(self) -> None:
|
| 276 |
+
"""Register worker with router with error handling.
|
| 277 |
+
|
| 278 |
+
This method attempts to register the current worker with a router service.
|
| 279 |
+
If registration fails, it logs an error but does not raise an exception,
|
| 280 |
+
allowing the server to continue operating without router integration.
|
| 281 |
+
|
| 282 |
+
Raises:
|
| 283 |
+
Does not raise exceptions - all errors are logged and handled gracefully.
|
| 284 |
+
"""
|
| 285 |
+
try:
|
| 286 |
+
url = f"http://{self.router_ip}:{self.router_port}/add_worker"
|
| 287 |
+
params = {"url": f"http://{self.server_args.host}:{self.server_args.port}"}
|
| 288 |
+
response = requests.post(url, params=params, timeout=self.timeout)
|
| 289 |
+
response.raise_for_status()
|
| 290 |
+
logger.info("Successfully registered with router")
|
| 291 |
+
except Exception as e:
|
| 292 |
+
logger.error(f"Failed to register with router: {e}")
|
| 293 |
+
# Don't raise here - server can still work without router
|
| 294 |
+
|
| 295 |
+
def _make_request(
|
| 296 |
+
self,
|
| 297 |
+
endpoint: str,
|
| 298 |
+
payload: Optional[dict[str, Any]] = None,
|
| 299 |
+
method: str = "POST",
|
| 300 |
+
timeout: float = DEFAULT_TIMEOUT,
|
| 301 |
+
only_master: bool = True,
|
| 302 |
+
) -> dict[str, Any]:
|
| 303 |
+
"""Make a HTTP request with retry logic and consistent error handling.
|
| 304 |
+
|
| 305 |
+
Args:
|
| 306 |
+
endpoint (str): The API endpoint to call (without leading slash)
|
| 307 |
+
payload (Optional[Dict[str, Any]], optional): The JSON payload to send.
|
| 308 |
+
Defaults to empty dict if None.
|
| 309 |
+
method (str, optional): HTTP method to use. Defaults to "POST".
|
| 310 |
+
|
| 311 |
+
Returns:
|
| 312 |
+
Dict[str, Any]: The JSON response from the server
|
| 313 |
+
|
| 314 |
+
Raises:
|
| 315 |
+
requests.HTTPError: If the HTTP request fails with a client/server error
|
| 316 |
+
RuntimeError: If all retry attempts are exhausted
|
| 317 |
+
|
| 318 |
+
Note:
|
| 319 |
+
- For non-master nodes (node_rank != 0), returns empty dict immediately
|
| 320 |
+
- Uses exponential backoff for retries
|
| 321 |
+
- Logs warnings for timeout and connection errors, errors for HTTP errors
|
| 322 |
+
"""
|
| 323 |
+
if only_master and self.node_rank != 0:
|
| 324 |
+
return {}
|
| 325 |
+
|
| 326 |
+
url = f"http://{self.server_args.host}:{self.server_args.port}/{endpoint}"
|
| 327 |
+
|
| 328 |
+
for attempt in range(self.max_attempts):
|
| 329 |
+
try:
|
| 330 |
+
if method.upper() == "GET":
|
| 331 |
+
response = requests.get(url, timeout=self.timeout)
|
| 332 |
+
else:
|
| 333 |
+
response = requests.post(url, json=payload or {}, timeout=self.timeout)
|
| 334 |
+
|
| 335 |
+
response.raise_for_status()
|
| 336 |
+
return _read_response(response)
|
| 337 |
+
|
| 338 |
+
except requests.exceptions.Timeout:
|
| 339 |
+
logger.warning(f"Request to {endpoint} timed out (attempt {attempt + 1})")
|
| 340 |
+
except requests.exceptions.ConnectionError:
|
| 341 |
+
logger.warning(f"Connection error for {endpoint} (attempt {attempt + 1})")
|
| 342 |
+
except requests.exceptions.HTTPError as e:
|
| 343 |
+
_log_error_body(endpoint, e.response.status_code, e.response.text)
|
| 344 |
+
raise
|
| 345 |
+
except Exception as e:
|
| 346 |
+
logger.error(f"Unexpected error for {endpoint}: {e}")
|
| 347 |
+
if attempt == self.max_attempts - 1:
|
| 348 |
+
raise
|
| 349 |
+
|
| 350 |
+
if attempt < self.max_attempts - 1:
|
| 351 |
+
time.sleep(self.retry_delay * (2**attempt))
|
| 352 |
+
|
| 353 |
+
raise RuntimeError(f"Failed to complete request to {endpoint} after {self.max_attempts} attempts")
|
| 354 |
+
|
| 355 |
+
def update_weights_from_tensor(self, req: UpdateWeightsFromTensorReqInput) -> dict[str, Any]:
|
| 356 |
+
"""Update model weights from tensor data.
|
| 357 |
+
|
| 358 |
+
The HTTP server will only post meta data, and the real weights will be
|
| 359 |
+
copied directly from GPUs.
|
| 360 |
+
|
| 361 |
+
Args:
|
| 362 |
+
serialized_named_tensors (List[str]): List of serialized tensor data
|
| 363 |
+
load_format (Optional[str], optional): Format specification for loading weights.
|
| 364 |
+
Defaults to None.
|
| 365 |
+
flush_cache (bool, optional): Whether to flush cache after updating weights.
|
| 366 |
+
Defaults to False.
|
| 367 |
+
|
| 368 |
+
Returns:
|
| 369 |
+
Dict[str, Any]: Server response containing update status
|
| 370 |
+
|
| 371 |
+
Note:
|
| 372 |
+
The model should be on GPUs rather than CPU for this functionality to work properly.
|
| 373 |
+
If you encounter issues, ensure your model is loaded on GPU devices rather than CPU.
|
| 374 |
+
"""
|
| 375 |
+
import base64
|
| 376 |
+
|
| 377 |
+
named_tensors = req.serialized_named_tensors
|
| 378 |
+
load_format = req.load_format
|
| 379 |
+
flush_cache = req.flush_cache
|
| 380 |
+
|
| 381 |
+
if named_tensors:
|
| 382 |
+
serialized_named_tensors = [
|
| 383 |
+
base64.b64encode(named_tensor).decode("utf-8") for named_tensor in named_tensors
|
| 384 |
+
]
|
| 385 |
+
else:
|
| 386 |
+
serialized_named_tensors = []
|
| 387 |
+
|
| 388 |
+
return self._make_request(
|
| 389 |
+
"update_weights_from_tensor",
|
| 390 |
+
{
|
| 391 |
+
"serialized_named_tensors": serialized_named_tensors,
|
| 392 |
+
"load_format": load_format,
|
| 393 |
+
"flush_cache": flush_cache,
|
| 394 |
+
},
|
| 395 |
+
)
|
| 396 |
+
|
| 397 |
+
def shutdown(self) -> None:
|
| 398 |
+
"""Shutdown the HTTP server and clean up resources.
|
| 399 |
+
|
| 400 |
+
This method performs the following cleanup operations:
|
| 401 |
+
1. Unregisters the worker from the router (if configured)
|
| 402 |
+
2. Terminates the server process tree
|
| 403 |
+
|
| 404 |
+
All operations are performed with error handling to ensure graceful shutdown
|
| 405 |
+
even if individual steps fail.
|
| 406 |
+
|
| 407 |
+
Note:
|
| 408 |
+
This method should be called when the adapter is no longer needed
|
| 409 |
+
to ensure proper cleanup of resources and processes.
|
| 410 |
+
"""
|
| 411 |
+
# Unregister from router
|
| 412 |
+
if self.router_ip and self.router_port:
|
| 413 |
+
try:
|
| 414 |
+
url = f"http://{self.router_ip}:{self.router_port}/remove_worker"
|
| 415 |
+
params = {"url": f"http://{self.server_args.host}:{self.server_args.port}"}
|
| 416 |
+
requests.post(url, params=params, timeout=5.0) # Short timeout for shutdown
|
| 417 |
+
logger.info("Successfully unregistered from router")
|
| 418 |
+
except Exception as e:
|
| 419 |
+
logger.warning(f"Failed to unregister from router: {e}")
|
| 420 |
+
|
| 421 |
+
# Kill server process
|
| 422 |
+
if hasattr(self, "process") and self.process is not None:
|
| 423 |
+
try:
|
| 424 |
+
kill_process_tree(self.process.pid)
|
| 425 |
+
logger.info("Server process terminated")
|
| 426 |
+
except Exception as e:
|
| 427 |
+
logger.error(f"Failed to terminate server process: {e}")
|
| 428 |
+
|
| 429 |
+
def generate(
|
| 430 |
+
self,
|
| 431 |
+
prompt: Optional[str] = None,
|
| 432 |
+
sampling_params: Optional[dict[str, Any]] = None,
|
| 433 |
+
input_ids: Optional[list[int]] = None,
|
| 434 |
+
image_data: Optional[Any] = None,
|
| 435 |
+
return_logprob: bool = False,
|
| 436 |
+
logprob_start_len: Optional[int] = None,
|
| 437 |
+
top_logprobs_num: Optional[int] = None,
|
| 438 |
+
token_ids_logprob: Optional[list[int]] = None,
|
| 439 |
+
lora_path: Optional[str] = None,
|
| 440 |
+
custom_logit_processor: Optional[Callable] = None,
|
| 441 |
+
) -> dict[str, Any]:
|
| 442 |
+
"""Generate text using the SGLang server.
|
| 443 |
+
|
| 444 |
+
Args:
|
| 445 |
+
prompt (Optional[str], optional): Text prompt for generation. Defaults to None.
|
| 446 |
+
sampling_params (Optional[Dict[str, Any]], optional): Parameters controlling
|
| 447 |
+
text generation sampling. Defaults to None.
|
| 448 |
+
input_ids (Optional[List[int]], optional): Alternative to prompt, direct token IDs input.
|
| 449 |
+
Defaults to None.
|
| 450 |
+
image_data (Optional[Any], optional): Image data for multimodal generation.
|
| 451 |
+
Defaults to None.
|
| 452 |
+
return_logprob (bool, optional): Whether to return log probabilities.
|
| 453 |
+
Defaults to False.
|
| 454 |
+
logprob_start_len (Optional[int], optional): Starting length for log probability calculation.
|
| 455 |
+
Defaults to None.
|
| 456 |
+
top_logprobs_num (Optional[int], optional): Number of top log probabilities to return.
|
| 457 |
+
Defaults to None.
|
| 458 |
+
token_ids_logprob (Optional[List[int]], optional): Specific token IDs for
|
| 459 |
+
log probability calculation. Defaults to None.
|
| 460 |
+
lora_path (Optional[str], optional): Path to LoRA adapter weights. Defaults to None.
|
| 461 |
+
custom_logit_processor (Optional[Callable], optional): Custom logit processing function.
|
| 462 |
+
Defaults to None.
|
| 463 |
+
|
| 464 |
+
Returns:
|
| 465 |
+
Dict[str, Any]: Generated text and associated metadata from the server
|
| 466 |
+
|
| 467 |
+
Note:
|
| 468 |
+
Either prompt or input_ids should be provided, but not both.
|
| 469 |
+
The response format depends on the server configuration and parameters.
|
| 470 |
+
"""
|
| 471 |
+
payload = {
|
| 472 |
+
"text": prompt,
|
| 473 |
+
"sampling_params": sampling_params,
|
| 474 |
+
"input_ids": input_ids,
|
| 475 |
+
"image_data": image_data,
|
| 476 |
+
"return_logprob": return_logprob,
|
| 477 |
+
"logprob_start_len": logprob_start_len,
|
| 478 |
+
"top_logprobs_num": top_logprobs_num,
|
| 479 |
+
"token_ids_logprob": token_ids_logprob,
|
| 480 |
+
"lora_path": lora_path,
|
| 481 |
+
"custom_logit_processor": custom_logit_processor,
|
| 482 |
+
}
|
| 483 |
+
# Filter out None values
|
| 484 |
+
payload = {k: v for k, v in payload.items() if v is not None}
|
| 485 |
+
|
| 486 |
+
return self._make_request("generate", payload, only_master=False)
|
| 487 |
+
|
| 488 |
+
def reward_score(
|
| 489 |
+
self,
|
| 490 |
+
prompt: Optional[str] = None,
|
| 491 |
+
input_ids: Optional[list[int]] = None,
|
| 492 |
+
image_data: Optional[Any] = None,
|
| 493 |
+
lora_path: Optional[str] = None,
|
| 494 |
+
) -> dict[str, Any]:
|
| 495 |
+
assert self.server_args.is_embedding, "Score is only supported for embedding models"
|
| 496 |
+
payload = {
|
| 497 |
+
"text": prompt,
|
| 498 |
+
"input_ids": input_ids,
|
| 499 |
+
"image_data": image_data,
|
| 500 |
+
"lora_path": lora_path,
|
| 501 |
+
}
|
| 502 |
+
# Filter out None values
|
| 503 |
+
payload = {k: v for k, v in payload.items() if v is not None}
|
| 504 |
+
|
| 505 |
+
return self._make_request("classify", payload, only_master=False)
|
| 506 |
+
|
| 507 |
+
def flush_cache(self) -> dict[str, Any]:
|
| 508 |
+
"""Flush the cache of the server.
|
| 509 |
+
|
| 510 |
+
This method repeatedly attempts to flush the server cache until successful.
|
| 511 |
+
The flush operation will not return status 200 when there are pending requests.
|
| 512 |
+
|
| 513 |
+
Returns:
|
| 514 |
+
Dict[str, Any]: Server response indicating cache flush status.
|
| 515 |
+
For non-master nodes, returns empty dict.
|
| 516 |
+
|
| 517 |
+
Note:
|
| 518 |
+
Uses retry logic with limited attempts (max_attempts * 2) to avoid infinite loops.
|
| 519 |
+
Each retry includes a delay to allow pending requests to complete.
|
| 520 |
+
"""
|
| 521 |
+
if self.node_rank != 0:
|
| 522 |
+
return {}
|
| 523 |
+
|
| 524 |
+
# Use retry logic with limited attempts to avoid infinite loops
|
| 525 |
+
for attempt in range(self.max_attempts * 2): # Allow more retries for cache flush
|
| 526 |
+
try:
|
| 527 |
+
response = requests.get(
|
| 528 |
+
f"http://{self.server_args.host}:{self.server_args.port}/flush_cache", timeout=self.timeout
|
| 529 |
+
)
|
| 530 |
+
if response.status_code == 200:
|
| 531 |
+
return _read_response(response)
|
| 532 |
+
except Exception as e:
|
| 533 |
+
logger.warning(f"Error flushing cache (attempt {attempt + 1}): {e}")
|
| 534 |
+
|
| 535 |
+
time.sleep(self.retry_delay)
|
| 536 |
+
|
| 537 |
+
logger.error("Failed to flush cache after maximum attempts")
|
| 538 |
+
return {}
|
| 539 |
+
|
| 540 |
+
def release_memory_occupation(self, tags: Optional[list[str]] = None) -> dict[str, Any]:
|
| 541 |
+
"""Release GPU memory occupation temporarily.
|
| 542 |
+
|
| 543 |
+
Args:
|
| 544 |
+
tags (Optional[List[str]], optional): List of tags to specify which memory to release.
|
| 545 |
+
If None, releases all memory. Defaults to None. ["weights", "kv_cache"]
|
| 546 |
+
|
| 547 |
+
Returns:
|
| 548 |
+
Dict[str, Any]: Server response indicating memory release status
|
| 549 |
+
"""
|
| 550 |
+
return self._make_request("release_memory_occupation", {"tags": tags})
|
| 551 |
+
|
| 552 |
+
def resume_memory_occupation(self, tags: Optional[list[str]] = None) -> dict[str, Any]:
|
| 553 |
+
"""Resume GPU memory occupation.
|
| 554 |
+
|
| 555 |
+
Args:
|
| 556 |
+
tags (Optional[List[str]], optional): List of tags to specify which memory to resume.
|
| 557 |
+
If None, resumes all memory. Defaults to None. ["weights", "kv_cache"]
|
| 558 |
+
|
| 559 |
+
Returns:
|
| 560 |
+
Dict[str, Any]: Server response indicating memory resume status
|
| 561 |
+
"""
|
| 562 |
+
return self._make_request("resume_memory_occupation", {"tags": tags})
|
| 563 |
+
|
| 564 |
+
def abort_request(self, rid: str = "", abort_all: bool = False) -> dict[str, Any]:
|
| 565 |
+
"""Abort a request.
|
| 566 |
+
|
| 567 |
+
Args:
|
| 568 |
+
rid (str): The ID of the request to abort
|
| 569 |
+
abort_all (bool, optional): Whether to abort all requests. Defaults to False.
|
| 570 |
+
|
| 571 |
+
Returns:
|
| 572 |
+
Dict[str, Any]: Server response indicating abort status
|
| 573 |
+
"""
|
| 574 |
+
return self._make_request("abort_request", {"rid": rid, "abort_all": abort_all})
|
| 575 |
+
|
| 576 |
+
|
| 577 |
+
class AsyncHttpServerAdapter(HttpServerAdapter):
|
| 578 |
+
"""Asynchronous HTTP-based adapter for SGLang engines.
|
| 579 |
+
|
| 580 |
+
This class inherits from HttpServerAdapter and adds async capabilities
|
| 581 |
+
for non-blocking HTTP requests to the SGLang server. It provides the same
|
| 582 |
+
functionality as the synchronous version but with async/await support.
|
| 583 |
+
|
| 584 |
+
The async adapter is useful when you need to make multiple concurrent requests
|
| 585 |
+
or integrate with async frameworks. It uses aiohttp for efficient async HTTP
|
| 586 |
+
communication and maintains connection pooling for better performance.
|
| 587 |
+
|
| 588 |
+
Attributes:
|
| 589 |
+
max_connections (int): Maximum number of connections in the connection pool
|
| 590 |
+
"""
|
| 591 |
+
|
| 592 |
+
def __init__(
|
| 593 |
+
self,
|
| 594 |
+
router_ip: Optional[str] = None,
|
| 595 |
+
router_port: Optional[int] = None,
|
| 596 |
+
timeout: float = DEFAULT_TIMEOUT,
|
| 597 |
+
max_attempts: int = DEFAULT_MAX_ATTEMPTS,
|
| 598 |
+
retry_delay: float = DEFAULT_RETRY_DELAY,
|
| 599 |
+
max_connections: int = DEFAULT_MAX_CONNECTIONS,
|
| 600 |
+
first_rank_in_node: bool = False,
|
| 601 |
+
launch_server: bool = True,
|
| 602 |
+
**kwargs: Any,
|
| 603 |
+
) -> None:
|
| 604 |
+
"""Initialize the async HTTP server engine adapter.
|
| 605 |
+
|
| 606 |
+
Args:
|
| 607 |
+
router_ip (Optional[str], optional): IP address of router for worker registration.
|
| 608 |
+
Defaults to None.
|
| 609 |
+
router_port (Optional[int], optional): Port of router for worker registration.
|
| 610 |
+
Defaults to None.
|
| 611 |
+
timeout (float, optional): HTTP request timeout in seconds.
|
| 612 |
+
Defaults to DEFAULT_TIMEOUT.
|
| 613 |
+
max_attempts (int, optional): Maximum number of retry attempts for failed requests.
|
| 614 |
+
Defaults to DEFAULT_MAX_ATTEMPTS.
|
| 615 |
+
retry_delay (float, optional): Base delay between retries in seconds.
|
| 616 |
+
Defaults to DEFAULT_RETRY_DELAY.
|
| 617 |
+
max_connections (int, optional): Maximum number of connections in the connection pool.
|
| 618 |
+
Defaults to DEFAULT_MAX_CONNECTIONS.
|
| 619 |
+
launch_server (bool, optional): Whether to launch the server process.
|
| 620 |
+
Defaults to True.
|
| 621 |
+
**kwargs (Any): Additional arguments passed to ServerArgs
|
| 622 |
+
"""
|
| 623 |
+
super().__init__(
|
| 624 |
+
router_ip,
|
| 625 |
+
router_port,
|
| 626 |
+
timeout,
|
| 627 |
+
max_attempts,
|
| 628 |
+
retry_delay,
|
| 629 |
+
first_rank_in_node,
|
| 630 |
+
launch_server=launch_server,
|
| 631 |
+
**kwargs,
|
| 632 |
+
)
|
| 633 |
+
self.max_connections: int = max_connections
|
| 634 |
+
|
| 635 |
+
@asynccontextmanager
|
| 636 |
+
async def _get_session(self) -> aiohttp.ClientSession:
|
| 637 |
+
"""Context manager for safe session access with proper connection pooling.
|
| 638 |
+
|
| 639 |
+
Yields:
|
| 640 |
+
aiohttp.ClientSession: Session instance for making HTTP requests
|
| 641 |
+
|
| 642 |
+
Note:
|
| 643 |
+
This method creates a new session for each request to avoid resource competition
|
| 644 |
+
while still maintaining proper connection pooling through the shared connector.
|
| 645 |
+
"""
|
| 646 |
+
# Create a new session for each request to avoid resource competition
|
| 647 |
+
connector = aiohttp.TCPConnector(
|
| 648 |
+
limit=self.max_connections,
|
| 649 |
+
limit_per_host=self.max_connections // 4,
|
| 650 |
+
ttl_dns_cache=300,
|
| 651 |
+
use_dns_cache=True,
|
| 652 |
+
)
|
| 653 |
+
timeout = aiohttp.ClientTimeout(total=self.timeout)
|
| 654 |
+
session = aiohttp.ClientSession(connector=connector, timeout=timeout)
|
| 655 |
+
|
| 656 |
+
try:
|
| 657 |
+
yield session
|
| 658 |
+
finally:
|
| 659 |
+
# Always close the session to free up resources
|
| 660 |
+
if not session.closed:
|
| 661 |
+
await session.close()
|
| 662 |
+
|
| 663 |
+
async def _make_async_request(
|
| 664 |
+
self,
|
| 665 |
+
endpoint: str,
|
| 666 |
+
payload: Optional[dict[str, Any]] = None,
|
| 667 |
+
method: str = "POST",
|
| 668 |
+
timeout: float = DEFAULT_TIMEOUT,
|
| 669 |
+
only_master: bool = True,
|
| 670 |
+
) -> dict[str, Any]:
|
| 671 |
+
"""Make an async HTTP request with retry logic and consistent error handling.
|
| 672 |
+
|
| 673 |
+
Args:
|
| 674 |
+
endpoint (str): The API endpoint to call (without leading slash)
|
| 675 |
+
payload (Optional[Dict[str, Any]], optional): The JSON payload to send.
|
| 676 |
+
Defaults to empty dict if None.
|
| 677 |
+
method (str, optional): HTTP method to use. Defaults to "POST".
|
| 678 |
+
|
| 679 |
+
Returns:
|
| 680 |
+
Dict[str, Any]: The JSON response from the server
|
| 681 |
+
|
| 682 |
+
Raises:
|
| 683 |
+
aiohttp.ClientResponseError: If the HTTP request fails with a client/server error
|
| 684 |
+
RuntimeError: If all retry attempts are exhausted
|
| 685 |
+
|
| 686 |
+
Note:
|
| 687 |
+
- For non-master nodes (node_rank != 0), returns empty dict immediately
|
| 688 |
+
- Uses exponential backoff for retries
|
| 689 |
+
- Logs warnings for timeout and connection errors, errors for HTTP errors
|
| 690 |
+
"""
|
| 691 |
+
if only_master and self.node_rank != 0:
|
| 692 |
+
return {}
|
| 693 |
+
|
| 694 |
+
url = f"http://{self.server_args.host}:{self.server_args.port}/{endpoint}"
|
| 695 |
+
|
| 696 |
+
for attempt in range(self.max_attempts):
|
| 697 |
+
try:
|
| 698 |
+
async with self._get_session() as session:
|
| 699 |
+
if method.upper() == "GET":
|
| 700 |
+
async with session.get(url, timeout=timeout) as response:
|
| 701 |
+
# Read the body here: it is gone once this block exits, and
|
| 702 |
+
# ClientResponseError does not carry it.
|
| 703 |
+
if response.status >= 400:
|
| 704 |
+
_log_error_body(endpoint, response.status, await response.text())
|
| 705 |
+
response.raise_for_status()
|
| 706 |
+
return await _read_async_response(response)
|
| 707 |
+
else:
|
| 708 |
+
async with session.post(url, json=payload or {}, timeout=timeout) as response:
|
| 709 |
+
if response.status >= 400:
|
| 710 |
+
_log_error_body(endpoint, response.status, await response.text())
|
| 711 |
+
response.raise_for_status()
|
| 712 |
+
return await _read_async_response(response)
|
| 713 |
+
|
| 714 |
+
except asyncio.TimeoutError:
|
| 715 |
+
logger.warning(f"Async request to {endpoint} timed out (attempt {attempt + 1})")
|
| 716 |
+
except aiohttp.ClientConnectorError:
|
| 717 |
+
logger.warning(f"Connection error for {endpoint} (attempt {attempt + 1})")
|
| 718 |
+
except aiohttp.ClientResponseError as e:
|
| 719 |
+
logger.error(f"HTTP error for {endpoint}: {e}")
|
| 720 |
+
raise
|
| 721 |
+
except Exception as e:
|
| 722 |
+
logger.error(f"Unexpected error for {endpoint}: {e}")
|
| 723 |
+
if attempt == self.max_attempts - 1:
|
| 724 |
+
raise
|
| 725 |
+
|
| 726 |
+
if attempt < self.max_attempts - 1:
|
| 727 |
+
await asyncio.sleep(self.retry_delay * (2**attempt))
|
| 728 |
+
|
| 729 |
+
raise RuntimeError(f"Failed to complete async request to {endpoint} after {self.max_attempts} attempts")
|
| 730 |
+
|
| 731 |
+
async def release_memory_occupation(self, tags: Optional[list[str]] = None) -> dict[str, Any]:
|
| 732 |
+
"""Release GPU memory occupation temporarily (async version).
|
| 733 |
+
|
| 734 |
+
Args:
|
| 735 |
+
tags (Optional[List[str]], optional): List of tags to specify which memory to release.
|
| 736 |
+
If None, releases all memory. Defaults to None. ["weights", "kv_cache"]
|
| 737 |
+
|
| 738 |
+
Returns:
|
| 739 |
+
Dict[str, Any]: Server response indicating memory release status
|
| 740 |
+
"""
|
| 741 |
+
return await self._make_async_request("release_memory_occupation", {"tags": tags})
|
| 742 |
+
|
| 743 |
+
async def resume_memory_occupation(self, tags: Optional[list[str]] = None) -> dict[str, Any]:
|
| 744 |
+
"""Resume GPU memory occupation (async version).
|
| 745 |
+
|
| 746 |
+
Similar to AsyncEngine, this method handles first-time weight reloading
|
| 747 |
+
by calling release_memory_occupation if needed.
|
| 748 |
+
|
| 749 |
+
Args:
|
| 750 |
+
tags (Optional[List[str]], optional): List of tags to specify which memory to resume.
|
| 751 |
+
If None, resumes all memory. Defaults to None. ["weights", "kv_cache"]
|
| 752 |
+
|
| 753 |
+
Returns:
|
| 754 |
+
Dict[str, Any]: Server response indicating memory resume status
|
| 755 |
+
"""
|
| 756 |
+
return await self._make_async_request("resume_memory_occupation", {"tags": tags})
|
| 757 |
+
|
| 758 |
+
async def update_weights_from_tensor(
|
| 759 |
+
self,
|
| 760 |
+
req: UpdateWeightsFromTensorReqInput,
|
| 761 |
+
) -> dict[str, Any]:
|
| 762 |
+
"""Update model weights from tensor data asynchronously.
|
| 763 |
+
|
| 764 |
+
Args:
|
| 765 |
+
serialized_named_tensors (List[str]): List of serialized tensor data
|
| 766 |
+
load_format (Optional[str], optional): Format specification for loading weights.
|
| 767 |
+
Defaults to None.
|
| 768 |
+
flush_cache (bool, optional): Whether to flush cache after updating weights.
|
| 769 |
+
Defaults to True.
|
| 770 |
+
|
| 771 |
+
Returns:
|
| 772 |
+
Dict[str, Any]: Server response containing update status
|
| 773 |
+
"""
|
| 774 |
+
import base64
|
| 775 |
+
|
| 776 |
+
named_tensors = req.serialized_named_tensors
|
| 777 |
+
load_format = req.load_format
|
| 778 |
+
flush_cache = req.flush_cache
|
| 779 |
+
|
| 780 |
+
serialized_named_tensors = [base64.b64encode(named_tensor).decode("utf-8") for named_tensor in named_tensors]
|
| 781 |
+
return await self._make_async_request(
|
| 782 |
+
"update_weights_from_tensor",
|
| 783 |
+
{
|
| 784 |
+
"serialized_named_tensors": serialized_named_tensors,
|
| 785 |
+
"load_format": load_format,
|
| 786 |
+
"flush_cache": flush_cache,
|
| 787 |
+
},
|
| 788 |
+
)
|
| 789 |
+
|
| 790 |
+
async def load_lora_adapter_from_tensor(self, req):
|
| 791 |
+
import base64
|
| 792 |
+
|
| 793 |
+
# Base64 like update_weights_from_tensor above: the field is List[bytes], JSON has none.
|
| 794 |
+
serialized_named_tensors = [base64.b64encode(t).decode("utf-8") for t in req.serialized_named_tensors]
|
| 795 |
+
return await self._make_async_request(
|
| 796 |
+
"load_lora_adapter_from_tensors",
|
| 797 |
+
{
|
| 798 |
+
"lora_name": req.lora_name,
|
| 799 |
+
"config_dict": req.config_dict,
|
| 800 |
+
"serialized_named_tensors": serialized_named_tensors,
|
| 801 |
+
},
|
| 802 |
+
)
|
| 803 |
+
|
| 804 |
+
async def unload_lora_adapter(self, lora_name: str):
|
| 805 |
+
return await self._make_async_request(
|
| 806 |
+
"unload_lora_adapter",
|
| 807 |
+
{
|
| 808 |
+
"lora_name": lora_name,
|
| 809 |
+
},
|
| 810 |
+
)
|
| 811 |
+
|
| 812 |
+
async def available_models(self):
|
| 813 |
+
return await self._make_async_request(endpoint="v1/models", method="GET")
|
| 814 |
+
|
| 815 |
+
async def flush_cache(self) -> dict[str, Any]:
|
| 816 |
+
"""Flush the cache of the server asynchronously.
|
| 817 |
+
|
| 818 |
+
Similar to the sync version, this method retries until the cache
|
| 819 |
+
is successfully flushed. It uses async sleep between retries.
|
| 820 |
+
|
| 821 |
+
Returns:
|
| 822 |
+
Dict[str, Any]: Server response indicating cache flush status.
|
| 823 |
+
For non-master nodes, returns empty dict.
|
| 824 |
+
|
| 825 |
+
Note:
|
| 826 |
+
Uses retry logic with limited attempts (max_attempts * 4) to avoid infinite loops.
|
| 827 |
+
Each retry includes an async delay to allow pending requests to complete.
|
| 828 |
+
"""
|
| 829 |
+
if self.node_rank != 0:
|
| 830 |
+
return {}
|
| 831 |
+
|
| 832 |
+
# Use retry logic with limited attempts to avoid infinite loops
|
| 833 |
+
for attempt in range(self.max_attempts * 4): # Allow more retries for cache flush
|
| 834 |
+
try:
|
| 835 |
+
async with self._get_session() as session:
|
| 836 |
+
url = f"http://{self.server_args.host}:{self.server_args.port}/flush_cache"
|
| 837 |
+
async with session.get(url) as response:
|
| 838 |
+
if response.status == 200:
|
| 839 |
+
return await _read_async_response(response)
|
| 840 |
+
except Exception as e:
|
| 841 |
+
logger.warning(f"Error flushing cache (attempt {attempt + 1}): {e}")
|
| 842 |
+
|
| 843 |
+
await asyncio.sleep(self.retry_delay)
|
| 844 |
+
|
| 845 |
+
logger.error("Failed to flush cache after maximum attempts")
|
| 846 |
+
return {}
|
| 847 |
+
|
| 848 |
+
async def generate(
|
| 849 |
+
self,
|
| 850 |
+
prompt: Optional[str] = None,
|
| 851 |
+
sampling_params: Optional[dict[str, Any]] = None,
|
| 852 |
+
input_ids: Optional[list[int]] = None,
|
| 853 |
+
image_data: Optional[Any] = None,
|
| 854 |
+
return_logprob: bool = False,
|
| 855 |
+
logprob_start_len: Optional[int] = None,
|
| 856 |
+
top_logprobs_num: Optional[int] = None,
|
| 857 |
+
token_ids_logprob: Optional[list[int]] = None,
|
| 858 |
+
lora_path: Optional[str] = None,
|
| 859 |
+
custom_logit_processor: Optional[Callable] = None,
|
| 860 |
+
) -> dict[str, Any]:
|
| 861 |
+
"""Generate text using the SGLang server asynchronously."""
|
| 862 |
+
logger.info("generate() started")
|
| 863 |
+
|
| 864 |
+
payload = {
|
| 865 |
+
"text": prompt,
|
| 866 |
+
"sampling_params": sampling_params,
|
| 867 |
+
"input_ids": input_ids,
|
| 868 |
+
"image_data": image_data,
|
| 869 |
+
"return_logprob": return_logprob,
|
| 870 |
+
"logprob_start_len": logprob_start_len,
|
| 871 |
+
"top_logprobs_num": top_logprobs_num,
|
| 872 |
+
"token_ids_logprob": token_ids_logprob,
|
| 873 |
+
"lora_path": lora_path,
|
| 874 |
+
"custom_logit_processor": custom_logit_processor,
|
| 875 |
+
}
|
| 876 |
+
|
| 877 |
+
# Filter out None values
|
| 878 |
+
payload = {k: v for k, v in payload.items() if v is not None}
|
| 879 |
+
|
| 880 |
+
# Send request
|
| 881 |
+
response = await self._make_async_request("generate", payload, timeout=self.timeout, only_master=False)
|
| 882 |
+
|
| 883 |
+
return response
|
| 884 |
+
|
| 885 |
+
async def async_generate(
|
| 886 |
+
self,
|
| 887 |
+
prompt: Optional[str] = None,
|
| 888 |
+
sampling_params: Optional[dict[str, Any]] = None,
|
| 889 |
+
input_ids: Optional[list[int]] = None,
|
| 890 |
+
image_data: Optional[Any] = None,
|
| 891 |
+
return_logprob: bool = False,
|
| 892 |
+
logprob_start_len: Optional[int] = None,
|
| 893 |
+
top_logprobs_num: Optional[int] = None,
|
| 894 |
+
token_ids_logprob: Optional[list[int]] = None,
|
| 895 |
+
lora_path: Optional[str] = None,
|
| 896 |
+
custom_logit_processor: Optional[Callable] = None,
|
| 897 |
+
) -> dict[str, Any]:
|
| 898 |
+
"""Async generate method that mirrors AsyncEngine.async_generate interface.
|
| 899 |
+
|
| 900 |
+
This method provides compatibility with AsyncEngine's async_generate method
|
| 901 |
+
by forwarding the call to the generate method. It ensures API consistency
|
| 902 |
+
between direct engine usage and HTTP-based engine usage.
|
| 903 |
+
|
| 904 |
+
Args:
|
| 905 |
+
prompt (Optional[str], optional): Text prompt for generation. Defaults to None.
|
| 906 |
+
sampling_params (Optional[Dict[str, Any]], optional): Parameters controlling
|
| 907 |
+
text generation sampling. Defaults to None.
|
| 908 |
+
input_ids (Optional[List[int]], optional): Alternative to prompt, direct token IDs input.
|
| 909 |
+
Defaults to None.
|
| 910 |
+
image_data (Optional[Any], optional): Image data for multimodal generation.
|
| 911 |
+
Defaults to None.
|
| 912 |
+
return_logprob (bool, optional): Whether to return log probabilities.
|
| 913 |
+
Defaults to False.
|
| 914 |
+
logprob_start_len (Optional[int], optional): Starting length for log probability calculation.
|
| 915 |
+
Defaults to None.
|
| 916 |
+
top_logprobs_num (Optional[int], optional): Number of top log probabilities to return.
|
| 917 |
+
Defaults to None.
|
| 918 |
+
token_ids_logprob (Optional[List[int]], optional): Specific token IDs for
|
| 919 |
+
log probability calculation. Defaults to None.
|
| 920 |
+
lora_path (Optional[str], optional): Path to LoRA adapter weights. Defaults to None.
|
| 921 |
+
custom_logit_processor (Optional[Callable], optional): Custom logit processing function.
|
| 922 |
+
Defaults to None.
|
| 923 |
+
|
| 924 |
+
Returns:
|
| 925 |
+
Dict[str, Any]: Generated text and associated metadata from the server
|
| 926 |
+
|
| 927 |
+
Note:
|
| 928 |
+
This method is provided for API compatibility with AsyncEngine.
|
| 929 |
+
It forwards all calls to the generate method.
|
| 930 |
+
"""
|
| 931 |
+
return await self.generate(
|
| 932 |
+
prompt=prompt,
|
| 933 |
+
sampling_params=sampling_params,
|
| 934 |
+
input_ids=input_ids,
|
| 935 |
+
image_data=image_data,
|
| 936 |
+
return_logprob=return_logprob,
|
| 937 |
+
logprob_start_len=logprob_start_len,
|
| 938 |
+
top_logprobs_num=top_logprobs_num,
|
| 939 |
+
token_ids_logprob=token_ids_logprob,
|
| 940 |
+
lora_path=lora_path,
|
| 941 |
+
custom_logit_processor=custom_logit_processor,
|
| 942 |
+
)
|
| 943 |
+
|
| 944 |
+
async def reward_score(
|
| 945 |
+
self,
|
| 946 |
+
prompt: Optional[str] = None,
|
| 947 |
+
input_ids: Optional[list[int]] = None,
|
| 948 |
+
image_data: Optional[Any] = None,
|
| 949 |
+
lora_path: Optional[str] = None,
|
| 950 |
+
) -> dict[str, Any]:
|
| 951 |
+
logger.info("reward_score() started")
|
| 952 |
+
payload = {
|
| 953 |
+
"text": prompt,
|
| 954 |
+
"input_ids": input_ids,
|
| 955 |
+
"image_data": image_data,
|
| 956 |
+
"lora_path": lora_path,
|
| 957 |
+
}
|
| 958 |
+
# Filter out None values
|
| 959 |
+
payload = {k: v for k, v in payload.items() if v is not None}
|
| 960 |
+
|
| 961 |
+
# Send request
|
| 962 |
+
response = await self._make_async_request("classify", payload, timeout=self.timeout, only_master=False)
|
| 963 |
+
|
| 964 |
+
return response
|
| 965 |
+
|
| 966 |
+
async def async_reward_score(
|
| 967 |
+
self,
|
| 968 |
+
prompt: Optional[str] = None,
|
| 969 |
+
input_ids: Optional[list[int]] = None,
|
| 970 |
+
image_data: Optional[Any] = None,
|
| 971 |
+
lora_path: Optional[str] = None,
|
| 972 |
+
) -> dict[str, Any]:
|
| 973 |
+
return await self.reward_score(
|
| 974 |
+
prompt=prompt,
|
| 975 |
+
input_ids=input_ids,
|
| 976 |
+
image_data=image_data,
|
| 977 |
+
lora_path=lora_path,
|
| 978 |
+
)
|
| 979 |
+
|
| 980 |
+
async def abort_request(self, rid: str = "", abort_all: bool = False) -> dict[str, Any]:
|
| 981 |
+
"""Abort a request asynchronously.
|
| 982 |
+
|
| 983 |
+
Args:
|
| 984 |
+
rid (str): The ID of the request to abort
|
| 985 |
+
abort_all (bool, optional): Whether to abort all requests. Defaults to False.
|
| 986 |
+
|
| 987 |
+
Returns:
|
| 988 |
+
Dict[str, Any]: Server response indicating abort status
|
| 989 |
+
"""
|
| 990 |
+
return await self._make_async_request("abort_request", {"rid": rid, "abort_all": abort_all})
|
repro/vendor/verl/workers/rollout/sglang_rollout/sglang_pd_replica.py
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 Bytedance Ltd. and/or its affiliates
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""SGLang PD-disaggregated replica: 1 prefill + N decode servers per replica,
|
| 15 |
+
asymmetric TP supported. MVP: prefill_replicas=1, whole replica on one node."""
|
| 16 |
+
|
| 17 |
+
import asyncio
|
| 18 |
+
import logging
|
| 19 |
+
import os
|
| 20 |
+
from dataclasses import replace as _dc_replace
|
| 21 |
+
from typing import Optional
|
| 22 |
+
|
| 23 |
+
import ray
|
| 24 |
+
from omegaconf import DictConfig
|
| 25 |
+
from ray.actor import ActorHandle
|
| 26 |
+
|
| 27 |
+
from verl.utils.device import is_torch_npu_available
|
| 28 |
+
from verl.utils.net_utils import get_free_port, is_valid_ipv6_address
|
| 29 |
+
from verl.workers.config import RolloutConfig
|
| 30 |
+
from verl.workers.rollout.sglang_rollout.async_sglang_server import (
|
| 31 |
+
SGLangReplica,
|
| 32 |
+
visible_devices_keyword,
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
logger = logging.getLogger(__file__)
|
| 36 |
+
logger.setLevel(logging.INFO)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class SGLangPDReplica(SGLangReplica):
|
| 40 |
+
"""Replica that runs SGLang in prefill-decode disaggregated mode."""
|
| 41 |
+
|
| 42 |
+
def __init__(
|
| 43 |
+
self,
|
| 44 |
+
replica_rank: int,
|
| 45 |
+
config: RolloutConfig,
|
| 46 |
+
model_config: DictConfig,
|
| 47 |
+
gpus_per_node: int = 8,
|
| 48 |
+
is_reward_model: bool = False,
|
| 49 |
+
is_teacher_model: bool = False,
|
| 50 |
+
):
|
| 51 |
+
super().__init__(
|
| 52 |
+
replica_rank,
|
| 53 |
+
config,
|
| 54 |
+
model_config,
|
| 55 |
+
gpus_per_node,
|
| 56 |
+
is_reward_model,
|
| 57 |
+
is_teacher_model,
|
| 58 |
+
)
|
| 59 |
+
disagg = self.config.disaggregation
|
| 60 |
+
assert disagg.enabled, "SGLangPDReplica requires rollout.disaggregation.enabled=True"
|
| 61 |
+
|
| 62 |
+
if disagg.prefill_replicas != 1:
|
| 63 |
+
raise NotImplementedError(f"prefill_replicas=1 only (got {disagg.prefill_replicas})")
|
| 64 |
+
self._n_prefill = disagg.prefill_replicas
|
| 65 |
+
self._n_decode = disagg.decode_replicas
|
| 66 |
+
|
| 67 |
+
self._prefill_tp = self.config.tensor_model_parallel_size
|
| 68 |
+
# Inline decode_tp default: OmegaConf/Ray serialization drops dataclass methods.
|
| 69 |
+
self._decode_tp = (
|
| 70 |
+
disagg.decode_tensor_model_parallel_size
|
| 71 |
+
if disagg.decode_tensor_model_parallel_size is not None
|
| 72 |
+
else self._prefill_tp
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
pd_world_size = self._prefill_tp + self._n_decode * self._decode_tp
|
| 76 |
+
if pd_world_size > gpus_per_node:
|
| 77 |
+
raise NotImplementedError(
|
| 78 |
+
f"PD replica needs {pd_world_size} GPUs but gpus_per_node={gpus_per_node}; "
|
| 79 |
+
f"use more replicas to span nodes"
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
if self.config.data_parallel_size != 1:
|
| 83 |
+
raise NotImplementedError(f"data_parallel_size=1 only (got {self.config.data_parallel_size})")
|
| 84 |
+
self.world_size = pd_world_size
|
| 85 |
+
self.gpus_per_replica_node = min(self.gpus_per_node, self.world_size)
|
| 86 |
+
assert self.world_size % self.gpus_per_replica_node == 0
|
| 87 |
+
self.nnodes = self.world_size // self.gpus_per_replica_node
|
| 88 |
+
|
| 89 |
+
self._prefill_servers: list[ActorHandle] = []
|
| 90 |
+
self._decode_servers: list[ActorHandle] = []
|
| 91 |
+
self._prefill_server_address: Optional[str] = None
|
| 92 |
+
self._decode_server_addresses: list[str] = []
|
| 93 |
+
self._bootstrap_port: Optional[int] = None
|
| 94 |
+
|
| 95 |
+
async def launch_servers(self):
|
| 96 |
+
assert len(self.workers) == self.world_size
|
| 97 |
+
assert not is_torch_npu_available(check_device=False), "PD on NPU not validated"
|
| 98 |
+
|
| 99 |
+
worker_infos = await asyncio.gather(
|
| 100 |
+
*[
|
| 101 |
+
worker.__ray_call__.remote(
|
| 102 |
+
lambda self: (
|
| 103 |
+
ray.get_runtime_context().get_node_id(),
|
| 104 |
+
os.environ[visible_devices_keyword],
|
| 105 |
+
)
|
| 106 |
+
)
|
| 107 |
+
for worker in self.workers
|
| 108 |
+
]
|
| 109 |
+
)
|
| 110 |
+
|
| 111 |
+
# Hold the bootstrap socket open until prefill binds it; closing earlier
|
| 112 |
+
# opens a TOCTOU window where another process can grab the port.
|
| 113 |
+
bootstrap_port = self.config.disaggregation.bootstrap_port
|
| 114 |
+
self._bootstrap_sock = None
|
| 115 |
+
if bootstrap_port is None:
|
| 116 |
+
prefill_host_ip = ray.util.get_node_ip_address().strip("[]")
|
| 117 |
+
bootstrap_port, self._bootstrap_sock = get_free_port(prefill_host_ip, with_alive_sock=True)
|
| 118 |
+
self._bootstrap_port = bootstrap_port
|
| 119 |
+
|
| 120 |
+
prefill_end = self._prefill_tp
|
| 121 |
+
prefill_workers = self.workers[0:prefill_end]
|
| 122 |
+
prefill_node_id = worker_infos[0][0]
|
| 123 |
+
prefill_devs = self._collect_cuda_devices(worker_infos[0:prefill_end])
|
| 124 |
+
|
| 125 |
+
if self._bootstrap_sock is not None:
|
| 126 |
+
self._bootstrap_sock.close()
|
| 127 |
+
self._bootstrap_sock = None
|
| 128 |
+
|
| 129 |
+
[prefill_server] = await self._launch_one(
|
| 130 |
+
role="prefill",
|
| 131 |
+
workers=prefill_workers,
|
| 132 |
+
node_id=prefill_node_id,
|
| 133 |
+
cuda_visible_devices=prefill_devs,
|
| 134 |
+
bootstrap_port=self._bootstrap_port,
|
| 135 |
+
tp=self._prefill_tp,
|
| 136 |
+
actor_name=f"sglang_server_{self.replica_rank}_0",
|
| 137 |
+
)
|
| 138 |
+
self._prefill_servers = [prefill_server]
|
| 139 |
+
|
| 140 |
+
prefill_address, prefill_port = await prefill_server.get_server_address.remote()
|
| 141 |
+
|
| 142 |
+
def _fmt(addr, port):
|
| 143 |
+
return f"[{addr}]:{port}" if is_valid_ipv6_address(addr) else f"{addr}:{port}"
|
| 144 |
+
|
| 145 |
+
self._prefill_server_address = _fmt(prefill_address, prefill_port)
|
| 146 |
+
|
| 147 |
+
self._decode_servers = []
|
| 148 |
+
self._decode_server_addresses = []
|
| 149 |
+
for i in range(self._n_decode):
|
| 150 |
+
start = self._prefill_tp + i * self._decode_tp
|
| 151 |
+
end = start + self._decode_tp
|
| 152 |
+
workers_i = self.workers[start:end]
|
| 153 |
+
node_id_i = worker_infos[start][0]
|
| 154 |
+
devs_i = self._collect_cuda_devices(worker_infos[start:end])
|
| 155 |
+
|
| 156 |
+
[decode_server] = await self._launch_one(
|
| 157 |
+
role="decode",
|
| 158 |
+
workers=workers_i,
|
| 159 |
+
node_id=node_id_i,
|
| 160 |
+
cuda_visible_devices=devs_i,
|
| 161 |
+
bootstrap_port=self._bootstrap_port,
|
| 162 |
+
tp=self._decode_tp,
|
| 163 |
+
actor_name=f"sglang_server_decode_{self.replica_rank}_{i}",
|
| 164 |
+
)
|
| 165 |
+
self._decode_servers.append(decode_server)
|
| 166 |
+
|
| 167 |
+
d_addr, d_port = await decode_server.get_server_address.remote()
|
| 168 |
+
self._decode_server_addresses.append(_fmt(d_addr, d_port))
|
| 169 |
+
|
| 170 |
+
self._server_address = self._prefill_server_address
|
| 171 |
+
self._server_handle = prefill_server
|
| 172 |
+
self.servers = list(self._prefill_servers) + list(self._decode_servers)
|
| 173 |
+
|
| 174 |
+
await prefill_server.set_pd_peer.remote(list(self._decode_servers), prefill_address)
|
| 175 |
+
|
| 176 |
+
logger.info(
|
| 177 |
+
f"SGLangPDReplica rank={self.replica_rank} launched: "
|
| 178 |
+
f"prefill={self._prefill_server_address}, "
|
| 179 |
+
f"decodes=[{', '.join(self._decode_server_addresses)}], "
|
| 180 |
+
f"bootstrap_port={self._bootstrap_port}"
|
| 181 |
+
)
|
| 182 |
+
|
| 183 |
+
@staticmethod
|
| 184 |
+
def _collect_cuda_devices(worker_infos) -> str:
|
| 185 |
+
devs = set()
|
| 186 |
+
for _, dev_str in worker_infos:
|
| 187 |
+
for d in dev_str.split(","):
|
| 188 |
+
if d.strip():
|
| 189 |
+
devs.add(int(d))
|
| 190 |
+
return ",".join(str(d) for d in sorted(devs))
|
| 191 |
+
|
| 192 |
+
async def _launch_one(
|
| 193 |
+
self,
|
| 194 |
+
role: str,
|
| 195 |
+
workers: list[ActorHandle],
|
| 196 |
+
node_id: str,
|
| 197 |
+
cuda_visible_devices: str,
|
| 198 |
+
bootstrap_port: int,
|
| 199 |
+
tp: int,
|
| 200 |
+
actor_name: str,
|
| 201 |
+
) -> list[ActorHandle]:
|
| 202 |
+
base_gpu_id = 0
|
| 203 |
+
if os.environ.get(f"RAY_EXPERIMENTAL_NOSET_{visible_devices_keyword}", None):
|
| 204 |
+
base_gpu_id = (0 + self.replica_rank * self.world_size) % self.gpus_per_node
|
| 205 |
+
|
| 206 |
+
pool_config = _dc_replace(self.config, tensor_model_parallel_size=tp)
|
| 207 |
+
|
| 208 |
+
server = self.server_class.options(
|
| 209 |
+
scheduling_strategy=ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy(
|
| 210 |
+
node_id=node_id, soft=False
|
| 211 |
+
),
|
| 212 |
+
runtime_env={"env_vars": {f"RAY_EXPERIMENTAL_NOSET_{visible_devices_keyword}": "1"}},
|
| 213 |
+
name=actor_name,
|
| 214 |
+
max_concurrency=self.max_concurrency,
|
| 215 |
+
).remote(
|
| 216 |
+
config=pool_config,
|
| 217 |
+
model_config=self.model_config,
|
| 218 |
+
rollout_mode=self.rollout_mode,
|
| 219 |
+
workers=workers,
|
| 220 |
+
replica_rank=self.replica_rank,
|
| 221 |
+
node_rank=0,
|
| 222 |
+
nnodes=1,
|
| 223 |
+
cuda_visible_devices=cuda_visible_devices,
|
| 224 |
+
base_gpu_id=base_gpu_id,
|
| 225 |
+
disaggregation_role=role,
|
| 226 |
+
disaggregation_bootstrap_port=bootstrap_port,
|
| 227 |
+
)
|
| 228 |
+
await server.launch_server.remote(master_address=None, master_port=None)
|
| 229 |
+
return [server]
|
repro/vendor/verl/workers/rollout/sglang_rollout/sglang_rollout.py
ADDED
|
@@ -0,0 +1,474 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2023-2024 SGLang Team
|
| 2 |
+
# Copyright 2025 ModelBest Inc. and/or its affiliates
|
| 3 |
+
# Copyright 2024 Bytedance Ltd. and/or its affiliates
|
| 4 |
+
#
|
| 5 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 6 |
+
# you may not use this file except in compliance with the License.
|
| 7 |
+
# You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 12 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 13 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 14 |
+
# See the License for the specific language governing permissions and
|
| 15 |
+
# limitations under the License.
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
|
| 18 |
+
import logging
|
| 19 |
+
import multiprocessing as mp
|
| 20 |
+
import os
|
| 21 |
+
from typing import Generator
|
| 22 |
+
|
| 23 |
+
import ray
|
| 24 |
+
import sglang.srt.entrypoints.engine
|
| 25 |
+
import torch
|
| 26 |
+
from sglang.srt.server_args import ServerArgs
|
| 27 |
+
from sglang.srt.utils import (
|
| 28 |
+
MultiprocessingSerializer,
|
| 29 |
+
assert_pkg_version,
|
| 30 |
+
is_cuda,
|
| 31 |
+
set_prometheus_multiproc_dir,
|
| 32 |
+
set_ulimit,
|
| 33 |
+
)
|
| 34 |
+
from sglang.srt.weight_sync.utils import _preprocess_tensor_for_update_weights
|
| 35 |
+
from sglang.srt.weight_sync.utils import update_weights as sgl_update_weights
|
| 36 |
+
from torch.distributed.device_mesh import DeviceMesh, init_device_mesh
|
| 37 |
+
|
| 38 |
+
from verl.utils.device import get_device_id
|
| 39 |
+
from verl.utils.net_utils import is_valid_ipv6_address
|
| 40 |
+
from verl.workers.config import HFModelConfig, RolloutConfig
|
| 41 |
+
from verl.workers.rollout.base import BaseRollout
|
| 42 |
+
from verl.workers.rollout.sglang_rollout.http_server_engine import AsyncHttpServerAdapter
|
| 43 |
+
from verl.workers.rollout.sglang_rollout.utils import (
|
| 44 |
+
DEEPSEEK_V4_FUSION_GROUPS,
|
| 45 |
+
SGLANG_LORA_NAME,
|
| 46 |
+
get_named_tensor_buckets,
|
| 47 |
+
lora_served_as_adapter,
|
| 48 |
+
normalize_peft_config_for_sglang,
|
| 49 |
+
)
|
| 50 |
+
|
| 51 |
+
logger = logging.getLogger(__file__)
|
| 52 |
+
logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _strip_lora_base_layer(name: str) -> str:
|
| 56 |
+
"""Drop the ``.base_layer`` segment ``replace_lora_wrapper()`` inserts for vLLM; SGLang
|
| 57 |
+
has no such level and its loader raises KeyError on it."""
|
| 58 |
+
return name.replace(".base_layer.", ".") if ".base_layer." in name else name
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _to_ipc_device(tensor: torch.Tensor) -> torch.Tensor:
|
| 62 |
+
"""Move a CPU tensor to the device, one at a time: sglang's IPC patch indexes a reducer slot
|
| 63 |
+
that only device tensors have."""
|
| 64 |
+
return tensor.to(get_device_id(), non_blocking=True) if tensor.device.type == "cpu" else tensor
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
# patch to avoid issue https://github.com/sgl-project/sglang/issues/6723
|
| 68 |
+
def _set_envs_and_config(server_args: ServerArgs):
|
| 69 |
+
# Set global environments
|
| 70 |
+
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
|
| 71 |
+
os.environ["NCCL_CUMEM_ENABLE"] = "0"
|
| 72 |
+
os.environ["NCCL_NVLS_ENABLE"] = str(int(server_args.enable_nccl_nvls))
|
| 73 |
+
os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1"
|
| 74 |
+
os.environ["CUDA_DEVICE_MAX_CONNECTIONS"] = "4"
|
| 75 |
+
os.environ["CUDA_MODULE_LOADING"] = "AUTO"
|
| 76 |
+
# Enable faulthandler in subprocesses
|
| 77 |
+
os.environ["PYTHONFAULTHANDLER"] = "1"
|
| 78 |
+
|
| 79 |
+
# Set prometheus env vars
|
| 80 |
+
if server_args.enable_metrics:
|
| 81 |
+
set_prometheus_multiproc_dir()
|
| 82 |
+
|
| 83 |
+
# Set ulimit
|
| 84 |
+
set_ulimit()
|
| 85 |
+
|
| 86 |
+
# Check flashinfer version
|
| 87 |
+
if server_args.attention_backend == "flashinfer":
|
| 88 |
+
assert_pkg_version(
|
| 89 |
+
"flashinfer_python",
|
| 90 |
+
"0.2.5",
|
| 91 |
+
"Please uninstall the old version and reinstall the latest version by following the instructions at https://docs.flashinfer.ai/installation.html.",
|
| 92 |
+
)
|
| 93 |
+
if is_cuda():
|
| 94 |
+
try:
|
| 95 |
+
# For sglang 0.5.12 and sglang_kernel > 0.4.2, naming is sglang_kernel
|
| 96 |
+
assert_pkg_version(
|
| 97 |
+
"sglang_kernel",
|
| 98 |
+
"0.1.1",
|
| 99 |
+
"Please reinstall the latest version with `pip install follow https://sgl-project.github.io/get_started/install.html#for-cuda-13`",
|
| 100 |
+
)
|
| 101 |
+
except Exception:
|
| 102 |
+
assert_pkg_version(
|
| 103 |
+
"sgl_kernel",
|
| 104 |
+
"0.1.1",
|
| 105 |
+
"Please reinstall the latest version with `pip install sgl-kernel --force-reinstall`",
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
# Set mp start method
|
| 109 |
+
mp.set_start_method("spawn", force=True)
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
sglang.srt.entrypoints.engine._set_envs_and_config = _set_envs_and_config
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
# because chatCompletion is an async method, it makes the whole ray actor be an async actor
|
| 116 |
+
# which can not call loop.run_until_complete. So we need to make the engine to be an async class
|
| 117 |
+
class ServerAdapter(BaseRollout):
|
| 118 |
+
"""SGLang server adapter used in native http server mode, serve as http client to request SGLang server
|
| 119 |
+
to resume/release/update weights and kv_cache.
|
| 120 |
+
|
| 121 |
+
- hybrid mode: reside in each hybrid worker to sync weights between training engine and SGLang server.
|
| 122 |
+
- standalone/colocated mode: just a dummy placeholder to occupy the GPU to prevent ray scheduling new GPU actor.
|
| 123 |
+
"""
|
| 124 |
+
|
| 125 |
+
def __init__(
|
| 126 |
+
self,
|
| 127 |
+
config: RolloutConfig,
|
| 128 |
+
model_config: HFModelConfig,
|
| 129 |
+
device_mesh: DeviceMesh,
|
| 130 |
+
replica_rank: int = -1,
|
| 131 |
+
):
|
| 132 |
+
super().__init__(config, model_config, device_mesh)
|
| 133 |
+
if self.config.get("quantization", None) == "fp8":
|
| 134 |
+
import sglang
|
| 135 |
+
from packaging import version
|
| 136 |
+
|
| 137 |
+
from verl.utils.sglang.sglang_fp8_utils import build_sglang_fp8_quant_config
|
| 138 |
+
|
| 139 |
+
assert version.parse(sglang.__version__) >= version.parse("0.5.5"), (
|
| 140 |
+
"sglang>=0.5.5 is required for FP8 quantization"
|
| 141 |
+
)
|
| 142 |
+
fp8_block_quant_kwargs = build_sglang_fp8_quant_config(self.model_config.hf_config)
|
| 143 |
+
self.model_config.hf_config.quantization_config = fp8_block_quant_kwargs
|
| 144 |
+
self._engine: AsyncHttpServerAdapter = None
|
| 145 |
+
|
| 146 |
+
rank = int(os.environ["RANK"])
|
| 147 |
+
local_world_size = int(os.environ["RAY_LOCAL_WORLD_SIZE"])
|
| 148 |
+
# PD asymmetric layout inflates per-replica footprint; must match
|
| 149 |
+
# agent_loop.py:_initialize_llm_servers or trainer-to-replica mapping breaks.
|
| 150 |
+
disagg = getattr(self.config, "disaggregation", None)
|
| 151 |
+
prefill_tp = self.config.tensor_model_parallel_size
|
| 152 |
+
if disagg is not None and getattr(disagg, "enabled", False):
|
| 153 |
+
# Inline decode_tp default: OmegaConf/Ray serialization drops dataclass methods.
|
| 154 |
+
decode_tp = (
|
| 155 |
+
disagg.decode_tensor_model_parallel_size
|
| 156 |
+
if disagg.decode_tensor_model_parallel_size is not None
|
| 157 |
+
else prefill_tp
|
| 158 |
+
)
|
| 159 |
+
rollout_world_size = (
|
| 160 |
+
(prefill_tp * disagg.prefill_replicas + decode_tp * disagg.decode_replicas)
|
| 161 |
+
* self.config.data_parallel_size
|
| 162 |
+
* self.config.pipeline_model_parallel_size
|
| 163 |
+
)
|
| 164 |
+
else:
|
| 165 |
+
rollout_world_size = prefill_tp * self.config.data_parallel_size * self.config.pipeline_model_parallel_size
|
| 166 |
+
if replica_rank == -1:
|
| 167 |
+
self.replica_rank = rank // rollout_world_size
|
| 168 |
+
else:
|
| 169 |
+
self.replica_rank = replica_rank
|
| 170 |
+
self.rollout_rank = rank % rollout_world_size
|
| 171 |
+
self.node_rank = self.rollout_rank // local_world_size
|
| 172 |
+
self.local_rank = self.rollout_rank % local_world_size
|
| 173 |
+
|
| 174 |
+
# Map each trainer rank to its co-located SGLang server so weight-update
|
| 175 |
+
# IPC handles stay on the GPU where they were created. Offset math
|
| 176 |
+
# assumes prefill_replicas == 1 (enforced by SGLangPDReplica); if that
|
| 177 |
+
# ever lifts, update both this block and SGLangPDReplica.launch_servers.
|
| 178 |
+
self._pd_role = None
|
| 179 |
+
self._pd_server_index = None
|
| 180 |
+
self._pd_tp_local_rank = None
|
| 181 |
+
if disagg is not None and getattr(disagg, "enabled", False):
|
| 182 |
+
decode_tp = (
|
| 183 |
+
disagg.decode_tensor_model_parallel_size
|
| 184 |
+
if disagg.decode_tensor_model_parallel_size is not None
|
| 185 |
+
else prefill_tp
|
| 186 |
+
)
|
| 187 |
+
# Modulo by single-group footprint so if DP>1 is ever enabled,
|
| 188 |
+
# each DP group's ranks resolve to the same role offsets.
|
| 189 |
+
footprint = prefill_tp + disagg.decode_replicas * decode_tp
|
| 190 |
+
local = self.rollout_rank % footprint
|
| 191 |
+
if local < prefill_tp:
|
| 192 |
+
self._pd_role = "prefill"
|
| 193 |
+
self._pd_server_index = 0
|
| 194 |
+
self._pd_tp_local_rank = local
|
| 195 |
+
else:
|
| 196 |
+
off = local - prefill_tp
|
| 197 |
+
self._pd_role = "decode"
|
| 198 |
+
self._pd_server_index = off // decode_tp
|
| 199 |
+
self._pd_tp_local_rank = off % decode_tp
|
| 200 |
+
self._has_server = (disagg is None or not getattr(disagg, "enabled", False)) or (self._pd_role is not None)
|
| 201 |
+
|
| 202 |
+
# sleep_level controls what gets released during sleep/release:
|
| 203 |
+
# 2 (default) = release weights + kv_cache (full sleep, merge path)
|
| 204 |
+
# 1 = release kv_cache only (keep base weights, adapter path)
|
| 205 |
+
# From config, not assigned after the first sync: sleep() branches on the same predicate.
|
| 206 |
+
self.sleep_level = 1 if lora_served_as_adapter(self.model_config) else 2
|
| 207 |
+
|
| 208 |
+
async def _init_server_adapter(self):
|
| 209 |
+
if self._engine is not None:
|
| 210 |
+
return
|
| 211 |
+
|
| 212 |
+
if not self._has_server:
|
| 213 |
+
return
|
| 214 |
+
|
| 215 |
+
# device_mesh is needed to gather cuda ipc handle to update weights.
|
| 216 |
+
if self.device_mesh is None:
|
| 217 |
+
assert torch.distributed.is_initialized(), "torch distributed must be initialized"
|
| 218 |
+
infer_tp = self.config.tensor_model_parallel_size * self.config.data_parallel_size
|
| 219 |
+
infer_pp = self.config.pipeline_model_parallel_size
|
| 220 |
+
infer_world_size = infer_tp * infer_pp
|
| 221 |
+
dp = torch.distributed.get_world_size() // infer_world_size
|
| 222 |
+
self.device_mesh = init_device_mesh(
|
| 223 |
+
"cpu", mesh_shape=(dp, infer_tp, infer_pp), mesh_dim_names=["dp", "infer_tp", "infer_pp"]
|
| 224 |
+
)
|
| 225 |
+
|
| 226 |
+
# Only the role's TP-rank-0 builds an adapter; others participate in
|
| 227 |
+
# FSDP collectives but skip HTTP dispatch.
|
| 228 |
+
if self._pd_role is not None:
|
| 229 |
+
if self._pd_tp_local_rank != 0:
|
| 230 |
+
return
|
| 231 |
+
else:
|
| 232 |
+
if self.device_mesh["infer_tp"].get_local_rank() != 0:
|
| 233 |
+
return
|
| 234 |
+
|
| 235 |
+
if self._pd_role == "prefill":
|
| 236 |
+
actor_name = f"sglang_server_{self.replica_rank}_0"
|
| 237 |
+
timeout_kwargs = {}
|
| 238 |
+
elif self._pd_role == "decode":
|
| 239 |
+
actor_name = f"sglang_server_decode_{self.replica_rank}_{self._pd_server_index}"
|
| 240 |
+
# Decode init on long-prompt workloads can stall past the default
|
| 241 |
+
# (60s × 12); shorter timeout + fewer attempts avoids trainer lockup.
|
| 242 |
+
timeout_kwargs = {"timeout": 10.0, "max_attempts": 2}
|
| 243 |
+
else:
|
| 244 |
+
actor_name = f"sglang_server_{self.replica_rank}_{self.node_rank}"
|
| 245 |
+
timeout_kwargs = {}
|
| 246 |
+
|
| 247 |
+
self.server_actor = ray.get_actor(actor_name)
|
| 248 |
+
server_address, server_port = await self.server_actor.get_server_address.remote()
|
| 249 |
+
host = f"[{server_address}]" if is_valid_ipv6_address(server_address) else server_address
|
| 250 |
+
logger.info(
|
| 251 |
+
f"ServerAdapter {self._pd_role or 'colocated'}: "
|
| 252 |
+
f"replica_rank={self.replica_rank}, rollout_rank={self.rollout_rank}, "
|
| 253 |
+
f"server={host}:{server_port}, actor={actor_name}"
|
| 254 |
+
)
|
| 255 |
+
|
| 256 |
+
self._engine = AsyncHttpServerAdapter(
|
| 257 |
+
model_path=self.model_config.local_path,
|
| 258 |
+
host=host,
|
| 259 |
+
port=server_port,
|
| 260 |
+
launch_server=False,
|
| 261 |
+
trust_remote_code=self.model_config.trust_remote_code,
|
| 262 |
+
**timeout_kwargs,
|
| 263 |
+
)
|
| 264 |
+
|
| 265 |
+
def _is_server_tp_leader(self) -> bool:
|
| 266 |
+
"""True if this rank is TP-rank-0 of its server's group.
|
| 267 |
+
|
| 268 |
+
In PD, the role's TP (prefill_tp or decode_tp) may differ from the
|
| 269 |
+
config-level TP that device_mesh was built with, so use
|
| 270 |
+
_pd_tp_local_rank when PD is active.
|
| 271 |
+
"""
|
| 272 |
+
if self._pd_role is not None:
|
| 273 |
+
return self._pd_tp_local_rank == 0
|
| 274 |
+
return self.device_mesh["infer_tp"].get_local_rank() == 0
|
| 275 |
+
|
| 276 |
+
async def resume(self, tags: list[str]):
|
| 277 |
+
"""Resume rollout weights or kv cache in GPU memory.
|
| 278 |
+
|
| 279 |
+
Args:
|
| 280 |
+
tag: weights or kv_cache.
|
| 281 |
+
"""
|
| 282 |
+
await self._init_server_adapter()
|
| 283 |
+
if self._engine is None:
|
| 284 |
+
return
|
| 285 |
+
if self._is_server_tp_leader() and self.config.free_cache_engine:
|
| 286 |
+
await self._engine.resume_memory_occupation(tags=tags)
|
| 287 |
+
|
| 288 |
+
async def release(self):
|
| 289 |
+
"""Release weights and kv cache in GPU memory.
|
| 290 |
+
|
| 291 |
+
When sleep_level=1 (LoRA adapter mode), only releases kv_cache
|
| 292 |
+
to keep base weights alive across training iterations.
|
| 293 |
+
When sleep_level=2 (default/merge mode), releases everything.
|
| 294 |
+
"""
|
| 295 |
+
await self._init_server_adapter()
|
| 296 |
+
if self._engine is None:
|
| 297 |
+
return
|
| 298 |
+
if self._is_server_tp_leader() and self.config.free_cache_engine:
|
| 299 |
+
if self.sleep_level == 1:
|
| 300 |
+
tags = ["kv_cache"]
|
| 301 |
+
else:
|
| 302 |
+
tags = ["kv_cache", "weights"]
|
| 303 |
+
await self._engine.release_memory_occupation(tags=tags)
|
| 304 |
+
|
| 305 |
+
async def update_weights(
|
| 306 |
+
self,
|
| 307 |
+
weights: Generator[tuple[str, torch.Tensor], None, None],
|
| 308 |
+
global_steps: int = None,
|
| 309 |
+
wire_format: str = "named_tensors",
|
| 310 |
+
**kwargs,
|
| 311 |
+
):
|
| 312 |
+
"""
|
| 313 |
+
Update model weights using tensor buckets, similar to THUDM/slime's implementation.
|
| 314 |
+
|
| 315 |
+
Notes:
|
| 316 |
+
- For the best performance of `rebuild_cuda_tensor`, it is recommended to:
|
| 317 |
+
1. Enable `RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES`.
|
| 318 |
+
2. Manually set `CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7`
|
| 319 |
+
when using Tensor Parallelism (TP >= 8).
|
| 320 |
+
- See reference implementations in SLIME:
|
| 321 |
+
- Main logic: https://github.com/THUDM/slime/blob/fb7605cc5fb09af0f9369d37f7192f12bddee577/slime/ray/ppo_actor.py#L452
|
| 322 |
+
- runtime envs: https://github.com/THUDM/slime/blob/fb7605cc5fb09af0f9369d37f7192f12bddee577/slime/ray/ppo_actor.py#L39
|
| 323 |
+
"""
|
| 324 |
+
await self._init_server_adapter()
|
| 325 |
+
|
| 326 |
+
# Delta checkpoint-engine wire (standalone replicas): the generator yields
|
| 327 |
+
# per-flush sparse payloads, applied in place through the verl custom
|
| 328 |
+
# weight loader. Hybrid replicas pass full (name, tensor) pairs with no
|
| 329 |
+
# wire_format kwarg and take the bucketed path below.
|
| 330 |
+
if wire_format == "delta_flush":
|
| 331 |
+
await self._update_weights_delta(weights, global_steps=global_steps)
|
| 332 |
+
return
|
| 333 |
+
|
| 334 |
+
# All ranks MUST iterate the weights generator below — DTensor.full_tensor()
|
| 335 |
+
# all_gather's across the FSDP group and skipping deadlocks the others.
|
| 336 |
+
# Only HTTP dispatch is gated on self._engine.
|
| 337 |
+
|
| 338 |
+
peft_config, base_sync_done = kwargs.get("peft_config", None), kwargs.get("base_sync_done", False)
|
| 339 |
+
if peft_config and base_sync_done:
|
| 340 |
+
if self.device_mesh["infer_tp"].get_local_rank() == 0:
|
| 341 |
+
# unload lora
|
| 342 |
+
models_result = await self._engine.available_models()
|
| 343 |
+
exists = any(item["id"] == SGLANG_LORA_NAME for item in models_result["data"])
|
| 344 |
+
if exists:
|
| 345 |
+
await self._engine.unload_lora_adapter(SGLANG_LORA_NAME)
|
| 346 |
+
|
| 347 |
+
# load lora by tensor
|
| 348 |
+
serialize_peft_config, serialize_named_tensors = self.wrap_lora_params(peft_config, weights)
|
| 349 |
+
from sglang.srt.managers.io_struct import LoadLoRAAdapterFromTensorsReqInput
|
| 350 |
+
|
| 351 |
+
req = LoadLoRAAdapterFromTensorsReqInput(
|
| 352 |
+
lora_name=SGLANG_LORA_NAME,
|
| 353 |
+
config_dict=serialize_peft_config,
|
| 354 |
+
serialized_named_tensors=serialize_named_tensors,
|
| 355 |
+
)
|
| 356 |
+
# send http request
|
| 357 |
+
await self._engine.load_lora_adapter_from_tensor(req)
|
| 358 |
+
else:
|
| 359 |
+
update_weights_bucket_bytes = int(self.config.checkpoint_engine.update_weights_bucket_megabytes) << 20
|
| 360 |
+
if self.config.get("quantization", None) == "fp8":
|
| 361 |
+
from verl.utils.sglang.sglang_fp8_utils import SGLangFP8QuantizerHelper
|
| 362 |
+
|
| 363 |
+
logger.info("Convert bf16 weights to fp8 format before loading")
|
| 364 |
+
fp8_quantizer_helper = SGLangFP8QuantizerHelper(self.model_config.hf_config.quantization_config)
|
| 365 |
+
weights = fp8_quantizer_helper.quant_weights_by_name(
|
| 366 |
+
weights,
|
| 367 |
+
dtype=self.model_config.hf_config.dtype,
|
| 368 |
+
)
|
| 369 |
+
else:
|
| 370 |
+
weights = weights
|
| 371 |
+
|
| 372 |
+
fusion_groups = (
|
| 373 |
+
DEEPSEEK_V4_FUSION_GROUPS
|
| 374 |
+
if getattr(self.model_config.hf_config, "model_type", None) == "deepseek_v4"
|
| 375 |
+
else ()
|
| 376 |
+
)
|
| 377 |
+
async for params_batch in get_named_tensor_buckets(
|
| 378 |
+
weights, update_weights_bucket_bytes, fusion_groups=fusion_groups
|
| 379 |
+
):
|
| 380 |
+
await sgl_update_weights(
|
| 381 |
+
engine=self._engine,
|
| 382 |
+
params_batch=[(_strip_lora_base_layer(name), _to_ipc_device(t)) for name, t in params_batch],
|
| 383 |
+
device_mesh_key="infer_tp",
|
| 384 |
+
device_mesh=self.device_mesh,
|
| 385 |
+
)
|
| 386 |
+
|
| 387 |
+
if self._engine is not None and self._is_server_tp_leader():
|
| 388 |
+
await self._engine.flush_cache()
|
| 389 |
+
if global_steps is not None:
|
| 390 |
+
await self.server_actor.set_global_steps.remote(global_steps)
|
| 391 |
+
|
| 392 |
+
async def _update_weights_delta(self, flushes, global_steps: int = None) -> None:
|
| 393 |
+
"""Apply the delta engine's sparse flushes in place.
|
| 394 |
+
|
| 395 |
+
``flushes`` is ``CheckpointEngine.receive_weights``'s generator of
|
| 396 |
+
``(named_tensors, is_last)`` items: one sentinel-encoded flush at a time,
|
| 397 |
+
already resident on this worker's GPU. Every rank of the replica consumes
|
| 398 |
+
the generator (the gather below is collective); TP0 posts one
|
| 399 |
+
``update_weights_from_tensor`` per flush with ``load_format`` pointing at
|
| 400 |
+
the verl delta loader, and the radix cache is flushed only on the last
|
| 401 |
+
flush of the stream.
|
| 402 |
+
"""
|
| 403 |
+
assert self._pd_role is None, "delta checkpoint engine does not support PD disaggregation"
|
| 404 |
+
applied = 0
|
| 405 |
+
for named, is_last in flushes:
|
| 406 |
+
await self._update_weights_delta_flush(named, flush_cache=is_last)
|
| 407 |
+
applied += 1
|
| 408 |
+
del named
|
| 409 |
+
if self._engine is not None and self._is_server_tp_leader() and global_steps is not None:
|
| 410 |
+
await self.server_actor.set_global_steps.remote(global_steps)
|
| 411 |
+
logger.info("delta apply v=%s flushes=%d (in-place via sglang loader)", global_steps, applied)
|
| 412 |
+
|
| 413 |
+
async def _update_weights_delta_flush(self, params_batch, flush_cache: bool) -> None:
|
| 414 |
+
"""Hand one sparse flush to the colocated SGLang server via same-GPU CUDA IPC.
|
| 415 |
+
|
| 416 |
+
SPMD across the replica's CheckpointEngineWorkers: every rank serializes its
|
| 417 |
+
local GPU copy, TP0 gathers the IPC handles and posts one
|
| 418 |
+
``update_weights_from_tensor`` with ``load_format`` pointing at the verl
|
| 419 |
+
loader. Same flow as ``sglang.srt.weight_sync.utils.update_weights``, inlined
|
| 420 |
+
so the radix cache is flushed only on the stream's last flush instead of on
|
| 421 |
+
every bucket. Checksum is re-verified inside each SGLang worker (fail loud).
|
| 422 |
+
"""
|
| 423 |
+
import torch.distributed as dist
|
| 424 |
+
from sglang.srt.managers.io_struct import UpdateWeightsFromTensorReqInput
|
| 425 |
+
from sglang.srt.utils import MultiprocessingSerializer
|
| 426 |
+
from sglang.srt.utils.patch_torch import monkey_patch_torch_reductions
|
| 427 |
+
|
| 428 |
+
try:
|
| 429 |
+
from sglang.srt.model_executor.model_runner import LocalSerializedTensor
|
| 430 |
+
except ImportError: # moved out of model_runner in sglang 0.5.16
|
| 431 |
+
from sglang.srt.model_executor.model_runner_components.weight_updater import LocalSerializedTensor
|
| 432 |
+
|
| 433 |
+
from verl.workers.rollout.sglang_rollout.delta_loader import LOADER_FQN
|
| 434 |
+
|
| 435 |
+
monkey_patch_torch_reductions()
|
| 436 |
+
mesh = self.device_mesh["infer_tp"]
|
| 437 |
+
tp_size = mesh.mesh.size()[0]
|
| 438 |
+
serialized = [(name, MultiprocessingSerializer.serialize(t.detach())) for name, t in params_batch]
|
| 439 |
+
|
| 440 |
+
gathered = [None for _ in range(tp_size)] if mesh.get_local_rank() == 0 else None
|
| 441 |
+
dist.gather_object(
|
| 442 |
+
obj=serialized,
|
| 443 |
+
object_gather_list=gathered,
|
| 444 |
+
dst=mesh.mesh.tolist()[0],
|
| 445 |
+
group=mesh.get_group(),
|
| 446 |
+
)
|
| 447 |
+
if mesh.get_local_rank() != 0:
|
| 448 |
+
return
|
| 449 |
+
|
| 450 |
+
named_tensors = [
|
| 451 |
+
(group[0][0], LocalSerializedTensor(values=[part[1] for part in group]))
|
| 452 |
+
for group in zip(*gathered, strict=True)
|
| 453 |
+
]
|
| 454 |
+
req = UpdateWeightsFromTensorReqInput(
|
| 455 |
+
serialized_named_tensors=[MultiprocessingSerializer.serialize(named_tensors) for _ in range(tp_size)],
|
| 456 |
+
load_format=LOADER_FQN,
|
| 457 |
+
flush_cache=flush_cache,
|
| 458 |
+
)
|
| 459 |
+
await self._engine.update_weights_from_tensor(req)
|
| 460 |
+
|
| 461 |
+
def wrap_lora_params(self, peft_config: dict, weights: Generator[tuple[str, torch.Tensor]]):
|
| 462 |
+
# peft config
|
| 463 |
+
peft_config_json = normalize_peft_config_for_sglang(peft_config)
|
| 464 |
+
|
| 465 |
+
# lora weights
|
| 466 |
+
processed_weights: dict[str, torch.Tensor] = {
|
| 467 |
+
name: _to_ipc_device(_preprocess_tensor_for_update_weights(tensor.detach())) for name, tensor in weights
|
| 468 |
+
}
|
| 469 |
+
|
| 470 |
+
# One serialized copy per TP rank, same field and shape as the weight sync above.
|
| 471 |
+
tp_size = self.device_mesh["infer_tp"].mesh.size()[0]
|
| 472 |
+
serialized_named_tensors = [MultiprocessingSerializer.serialize(processed_weights) for _ in range(tp_size)]
|
| 473 |
+
|
| 474 |
+
return peft_config_json, serialized_named_tensors
|
repro/vendor/verl/workers/rollout/sglang_rollout/utils.py
ADDED
|
@@ -0,0 +1,235 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2023-2024 SGLang Team
|
| 2 |
+
# Copyright 2025 ModelBest Inc. and/or its affiliates
|
| 3 |
+
#
|
| 4 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 5 |
+
# you may not use this file except in compliance with the License.
|
| 6 |
+
# You may obtain a copy of the License at
|
| 7 |
+
#
|
| 8 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 9 |
+
#
|
| 10 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 11 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 12 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 13 |
+
# See the License for the specific language governing permissions and
|
| 14 |
+
# limitations under the License.
|
| 15 |
+
|
| 16 |
+
import pickle
|
| 17 |
+
from typing import Any, Iterator, Optional
|
| 18 |
+
|
| 19 |
+
import numpy as np
|
| 20 |
+
import torch
|
| 21 |
+
import torch.distributed as dist
|
| 22 |
+
|
| 23 |
+
from verl.utils.device import get_device_name
|
| 24 |
+
from verl.workers.rollout.utils import ensure_async_iterator
|
| 25 |
+
|
| 26 |
+
SGLANG_LORA_NAME = "verl_actor_lora_name"
|
| 27 |
+
|
| 28 |
+
_DEEPSEEK_V4_FUSION_MEMBERS = (
|
| 29 |
+
("wq_a.weight", "wkv.weight"),
|
| 30 |
+
("wq_a.scale", "wkv.scale"),
|
| 31 |
+
("wq_a.weight_scale_inv", "wkv.weight_scale_inv"),
|
| 32 |
+
("compressor.wkv.weight", "compressor.wgate.weight"),
|
| 33 |
+
("indexer.compressor.wkv.weight", "indexer.compressor.wgate.weight"),
|
| 34 |
+
)
|
| 35 |
+
DEEPSEEK_V4_FUSION_GROUPS = tuple(
|
| 36 |
+
tuple(f".{attention}.{member}" for member in members)
|
| 37 |
+
for attention in ("self_attn", "attn")
|
| 38 |
+
for members in _DEEPSEEK_V4_FUSION_MEMBERS
|
| 39 |
+
)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def _fusion_key(name: str, groups: tuple[tuple[str, ...], ...]):
|
| 43 |
+
matches = [
|
| 44 |
+
(index, suffix, len(group)) for index, group in enumerate(groups) for suffix in group if name.endswith(suffix)
|
| 45 |
+
]
|
| 46 |
+
if len(matches) > 1:
|
| 47 |
+
raise ValueError(f"{name!r} matches multiple fusion groups: {matches}")
|
| 48 |
+
if not matches:
|
| 49 |
+
return None
|
| 50 |
+
index, suffix, size = matches[0]
|
| 51 |
+
return name[: -len(suffix)], index, size
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def normalize_peft_config_for_sglang(peft_config: dict) -> dict:
|
| 55 |
+
"""Normalize an engine's adapter config (enums to strings) for SGLang's adapter loader."""
|
| 56 |
+
normalized = dict(peft_config)
|
| 57 |
+
for key in ("task_type", "peft_type"):
|
| 58 |
+
if key in normalized:
|
| 59 |
+
normalized[key] = getattr(normalized[key], "value", normalized[key])
|
| 60 |
+
if "peft_type" not in normalized:
|
| 61 |
+
raise ValueError(
|
| 62 |
+
"adapter config has no 'peft_type', which SGLang's adapter loader requires. See "
|
| 63 |
+
"BaseEngine.get_per_tensor_param for the keys. Keys present: " + ", ".join(sorted(normalized))
|
| 64 |
+
)
|
| 65 |
+
# A bare string must stay one: list() would tear "all-linear" into characters.
|
| 66 |
+
target_modules = normalized["target_modules"]
|
| 67 |
+
normalized["target_modules"] = target_modules if isinstance(target_modules, str) else list(target_modules)
|
| 68 |
+
return normalized
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def lora_rank_of(model_config) -> int:
|
| 72 |
+
"""The LoRA rank, from whichever block carries it: megatron sets ``model.lora.rank``, fsdp
|
| 73 |
+
the flat ``model.lora_rank``."""
|
| 74 |
+
return max(int(getattr(model_config, "lora_rank", 0) or 0), int(model_config.lora.get("rank", 0) or 0))
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def lora_served_as_adapter(model_config) -> bool:
|
| 78 |
+
"""Whether SGLang should serve LoRA as a hot-swappable adapter.
|
| 79 |
+
|
| 80 |
+
``HFModelConfig`` carries two LoRA config blocks that are never synced: megatron
|
| 81 |
+
runs set ``model.lora.rank`` (dict) while fsdp runs set the flat ``model.lora_rank``,
|
| 82 |
+
so both must be checked to detect that LoRA is enabled at all.
|
| 83 |
+
|
| 84 |
+
With ``model.lora.merge=True`` the trainer merges the adapter into the base weights
|
| 85 |
+
and pushes a full HF-keyed weight update (``peft_config=None``), so no adapter is ever
|
| 86 |
+
loaded into SGLang: the engine must not be launched with ``enable_lora`` and requests
|
| 87 |
+
must not carry a ``lora_path``.
|
| 88 |
+
"""
|
| 89 |
+
lora_enabled = lora_rank_of(model_config) > 0
|
| 90 |
+
return lora_enabled and not model_config.lora.get("merge", False)
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def sglang_lora_target_modules(target_modules: Any) -> list[str]:
|
| 94 |
+
"""Render verl's ``model.target_modules`` as SGLang's ``lora_target_modules``."""
|
| 95 |
+
if target_modules == "all-linear":
|
| 96 |
+
return ["all"]
|
| 97 |
+
if isinstance(target_modules, str):
|
| 98 |
+
raise ValueError(
|
| 99 |
+
f"SGLang cannot serve a regex `target_modules` ({target_modules!r}); PEFT matches it "
|
| 100 |
+
f"against the whole parameter key. Use `all-linear`, or list the module names."
|
| 101 |
+
)
|
| 102 |
+
return list(target_modules)
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def broadcast_pyobj(
|
| 106 |
+
data: list[Any],
|
| 107 |
+
rank: int,
|
| 108 |
+
dist_group: Optional[torch.distributed.ProcessGroup] = None,
|
| 109 |
+
src: int = 0,
|
| 110 |
+
force_cpu_device: bool = False,
|
| 111 |
+
):
|
| 112 |
+
"""from https://github.com/sgl-project/sglang/blob/844e2f227ab0cce6ef818a719170ce37b9eb1e1b/python/sglang/srt/utils.py#L905
|
| 113 |
+
|
| 114 |
+
Broadcast inputs from src rank to all other ranks with torch.dist backend.
|
| 115 |
+
The `rank` here refer to the source rank on global process group (regardless
|
| 116 |
+
of dist_group argument).
|
| 117 |
+
"""
|
| 118 |
+
device = torch.device(get_device_name() if not force_cpu_device else "cpu")
|
| 119 |
+
|
| 120 |
+
if rank == src:
|
| 121 |
+
if len(data) == 0:
|
| 122 |
+
tensor_size = torch.tensor([0], dtype=torch.long, device=device)
|
| 123 |
+
dist.broadcast(tensor_size, src=src, group=dist_group)
|
| 124 |
+
else:
|
| 125 |
+
serialized_data = pickle.dumps(data)
|
| 126 |
+
size = len(serialized_data)
|
| 127 |
+
|
| 128 |
+
tensor_data = torch.ByteTensor(np.frombuffer(serialized_data, dtype=np.uint8)).to(device)
|
| 129 |
+
tensor_size = torch.tensor([size], dtype=torch.long, device=device)
|
| 130 |
+
|
| 131 |
+
dist.broadcast(tensor_size, src=src, group=dist_group)
|
| 132 |
+
dist.broadcast(tensor_data, src=src, group=dist_group)
|
| 133 |
+
return data
|
| 134 |
+
else:
|
| 135 |
+
tensor_size = torch.tensor([0], dtype=torch.long, device=device)
|
| 136 |
+
dist.broadcast(tensor_size, src=src, group=dist_group)
|
| 137 |
+
size = tensor_size.item()
|
| 138 |
+
|
| 139 |
+
if size == 0:
|
| 140 |
+
return []
|
| 141 |
+
|
| 142 |
+
tensor_data = torch.empty(size, dtype=torch.uint8, device=device)
|
| 143 |
+
dist.broadcast(tensor_data, src=src, group=dist_group)
|
| 144 |
+
|
| 145 |
+
serialized_data = bytes(tensor_data.cpu().numpy())
|
| 146 |
+
data = pickle.loads(serialized_data)
|
| 147 |
+
return data
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def _compact_for_bucket(tensor: torch.Tensor) -> torch.Tensor:
|
| 151 |
+
"""Return a tensor safe to retain in a weight-sync bucket without pinning extra memory.
|
| 152 |
+
|
| 153 |
+
``get_named_tensor_buckets`` keeps every tensor alive until its bucket is flushed. A tensor
|
| 154 |
+
that is a *view* into a larger backing buffer would therefore keep that whole buffer resident
|
| 155 |
+
(and ship the whole buffer downstream), so such views must be compacted with ``clone()``.
|
| 156 |
+
|
| 157 |
+
However the weights synced here come from ``DTensor.full_tensor()`` (a fresh all-gather) and
|
| 158 |
+
already own tight, contiguous storage. Cloning those allocates a second full-size buffer and
|
| 159 |
+
transiently doubles the tensor's footprint -- which OOMs on multi-GiB fused MoE weights
|
| 160 |
+
(e.g. ``[num_experts, ...]`` ``gate_up_proj``/``qkv``) while the actor params and rollout
|
| 161 |
+
weights are both already resident. Skip the clone when the tensor already owns its storage.
|
| 162 |
+
"""
|
| 163 |
+
return tensor.clone() if tensor._base is not None else tensor
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
async def get_named_tensor_buckets(
|
| 167 |
+
iterable: Iterator[tuple[str, torch.Tensor]],
|
| 168 |
+
bucket_bytes: int,
|
| 169 |
+
fusion_groups: tuple[tuple[str, ...], ...] = (),
|
| 170 |
+
) -> Iterator[list[tuple[str, torch.Tensor]]]:
|
| 171 |
+
"""
|
| 172 |
+
Group tensors into buckets based on a specified size in megabytes.
|
| 173 |
+
|
| 174 |
+
Args:
|
| 175 |
+
iterable: An iterator of tuples containing tensor names and tensors.
|
| 176 |
+
bucket_bytes: The maximum size of each bucket in bytes.
|
| 177 |
+
|
| 178 |
+
Yields:
|
| 179 |
+
Lists of tuples, where each tuple contains a tensor name and its corresponding tensor.
|
| 180 |
+
|
| 181 |
+
Example:
|
| 182 |
+
>>> tensors = [('tensor1', torch.randn(1000, 1000)), ('tensor2', torch.randn(2000, 2000))]
|
| 183 |
+
>>> for bucket in get_named_tensor_buckets(tensors, bucket_size_mb=10):
|
| 184 |
+
... print(bucket)
|
| 185 |
+
[('tensor1', tensor(...)), ('tensor2', tensor(...))]
|
| 186 |
+
|
| 187 |
+
"""
|
| 188 |
+
if bucket_bytes <= 0:
|
| 189 |
+
raise ValueError(f"bucket_bytes must be greater than 0, got {bucket_bytes}")
|
| 190 |
+
|
| 191 |
+
current_bucket = []
|
| 192 |
+
current_size = 0
|
| 193 |
+
# Members of a fused destination param (SGLang's DSv4 wqkv_a / wkv_gate) are
|
| 194 |
+
# held here until the group is whole, then placed in one bucket: that loader
|
| 195 |
+
# cats both halves inside a single load_weights call and asserts its cache
|
| 196 |
+
# empty on return, so a bucket boundary between them makes it fire. A full
|
| 197 |
+
# sync contains every member by construction, so this only ever delays a
|
| 198 |
+
# tensor by its sibling -- nothing is dropped.
|
| 199 |
+
staged: dict = {}
|
| 200 |
+
|
| 201 |
+
def _place(entries):
|
| 202 |
+
"""Put an indivisible run of entries in a bucket, cutting before it if
|
| 203 |
+
it would not fit rather than partway through."""
|
| 204 |
+
nonlocal current_bucket, current_size
|
| 205 |
+
total = sum(sz for _, _, sz in entries)
|
| 206 |
+
if current_bucket and current_size + total > bucket_bytes:
|
| 207 |
+
out, current_bucket, current_size = current_bucket, [], 0
|
| 208 |
+
return out, entries
|
| 209 |
+
return None, entries
|
| 210 |
+
|
| 211 |
+
async for name, tensor in ensure_async_iterator(iterable):
|
| 212 |
+
tensor_size = tensor.element_size() * tensor.numel()
|
| 213 |
+
match = _fusion_key(name, fusion_groups)
|
| 214 |
+
if match is not None:
|
| 215 |
+
prefix, group_index, expected_size = match
|
| 216 |
+
key = (prefix, group_index)
|
| 217 |
+
slot = staged.setdefault(key, [])
|
| 218 |
+
slot.append((name, _compact_for_bucket(tensor), tensor_size))
|
| 219 |
+
if len(slot) < expected_size:
|
| 220 |
+
continue
|
| 221 |
+
entries = staged.pop(key)
|
| 222 |
+
else:
|
| 223 |
+
entries = [(name, _compact_for_bucket(tensor), tensor_size)]
|
| 224 |
+
|
| 225 |
+
flushed, entries = _place(entries)
|
| 226 |
+
if flushed:
|
| 227 |
+
yield flushed
|
| 228 |
+
for n, t, sz in entries:
|
| 229 |
+
current_bucket.append((n, t))
|
| 230 |
+
current_size += sz
|
| 231 |
+
|
| 232 |
+
if staged:
|
| 233 |
+
raise ValueError(f"fused parameter groups never completed in this sync: {sorted(staged)}")
|
| 234 |
+
if current_bucket:
|
| 235 |
+
yield current_bucket
|
repro/vendor/verl/workers/rollout/trtllm_rollout/__init__.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 Bytedance Ltd. and/or its affiliates
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
repro/vendor/verl/workers/rollout/trtllm_rollout/trtllm_async_server.py
ADDED
|
@@ -0,0 +1,679 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 Bytedance Ltd. and/or its affiliates
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
import asyncio
|
| 15 |
+
import logging
|
| 16 |
+
import os
|
| 17 |
+
from typing import Any, Optional
|
| 18 |
+
|
| 19 |
+
import ray
|
| 20 |
+
import torch
|
| 21 |
+
from omegaconf import DictConfig
|
| 22 |
+
from ray.actor import ActorHandle
|
| 23 |
+
from ray.util import placement_group_table
|
| 24 |
+
from ray.util.placement_group import PlacementGroup
|
| 25 |
+
|
| 26 |
+
from verl.plugin.platform import get_platform
|
| 27 |
+
from verl.single_controller.ray import SubRayResourcePool
|
| 28 |
+
from verl.utils.config import omega_conf_to_dataclass
|
| 29 |
+
from verl.utils.net_utils import is_valid_ipv6_address
|
| 30 |
+
from verl.utils.profiler import (
|
| 31 |
+
DistProfiler,
|
| 32 |
+
build_rollout_dist_profiler,
|
| 33 |
+
relocate_rollout_traces,
|
| 34 |
+
rollout_profiler_global_ranks,
|
| 35 |
+
)
|
| 36 |
+
from verl.workers.config import HFModelConfig, RolloutConfig
|
| 37 |
+
from verl.workers.rollout.replica import RolloutMode, RolloutReplica, TokenOutput
|
| 38 |
+
from verl.workers.rollout.utils import get_max_position_embeddings, qwen2_5_vl_dedup_image_tokens, run_uvicorn
|
| 39 |
+
|
| 40 |
+
logger = logging.getLogger(__file__)
|
| 41 |
+
logger.setLevel(logging.INFO)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def _resolve_chat_stop_tokens(model_config) -> tuple[int, list[int]]:
|
| 45 |
+
"""Return (end_id, stop_token_ids) for TorchSampler.
|
| 46 |
+
|
| 47 |
+
Both TRTLLM's samplers stops only on end_id. For chat-format prompts the model
|
| 48 |
+
naturally ends each assistant turn with a chat-end token (e.g. <|im_end|>
|
| 49 |
+
for Qwen, <|eot_id|> for Llama-3) that is *different* from the base-model
|
| 50 |
+
eos_token_id. If end_id is set to the base eos the sampler ignores the
|
| 51 |
+
chat-end token and the model loops into a second turn, inflating response
|
| 52 |
+
lengths until max_tokens is hit.
|
| 53 |
+
|
| 54 |
+
For models without a distinct chat-end token the return values are
|
| 55 |
+
identical to the current default (end_id = hf_config.eos_token_id).
|
| 56 |
+
"""
|
| 57 |
+
eos_token_id = model_config.hf_config.eos_token_id
|
| 58 |
+
all_stop_ids: list[int] = list(eos_token_id) if isinstance(eos_token_id, list) else [eos_token_id]
|
| 59 |
+
|
| 60 |
+
if model_config.generation_config is not None:
|
| 61 |
+
gen_eos = model_config.generation_config.eos_token_id
|
| 62 |
+
if gen_eos is not None:
|
| 63 |
+
for t in gen_eos if isinstance(gen_eos, list) else [gen_eos]:
|
| 64 |
+
if t not in all_stop_ids:
|
| 65 |
+
all_stop_ids.append(t)
|
| 66 |
+
|
| 67 |
+
chat_end_id = None
|
| 68 |
+
if model_config.tokenizer is not None:
|
| 69 |
+
_chat_stop_strings = ["<|im_end|>", "<|eot_id|>", "<|end_of_turn|>"]
|
| 70 |
+
_added_vocab = model_config.tokenizer.get_added_vocab()
|
| 71 |
+
for stop_str in _chat_stop_strings:
|
| 72 |
+
if stop_str in _added_vocab:
|
| 73 |
+
tid = _added_vocab[stop_str]
|
| 74 |
+
if tid not in all_stop_ids:
|
| 75 |
+
all_stop_ids.append(tid)
|
| 76 |
+
if chat_end_id is None:
|
| 77 |
+
chat_end_id = tid
|
| 78 |
+
|
| 79 |
+
primary_end_id = chat_end_id if chat_end_id is not None else eos_token_id
|
| 80 |
+
logger.warning(f"TRT-LLM stop token IDs: {all_stop_ids}, end_id: {primary_end_id}")
|
| 81 |
+
return primary_end_id, all_stop_ids
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
@ray.remote
|
| 85 |
+
class TRTLLMHttpServer:
|
| 86 |
+
"""TensorRT LLM HTTP server in single node.
|
| 87 |
+
|
| 88 |
+
Args:
|
| 89 |
+
config (DictConfig): full config.
|
| 90 |
+
model_config (HFModelConfig): model config.
|
| 91 |
+
is_reward_model (bool): whether this is a reward model.
|
| 92 |
+
rollout_mode (RolloutMode): rollout mode.
|
| 93 |
+
workers (list[ActorHandle]): list of rollout workers.
|
| 94 |
+
replica_rank (int): replica rank, a replica may contain multiple nodes.
|
| 95 |
+
max_colocate_count (int): max colocate count.
|
| 96 |
+
pgs (list[PlacementGroup]): placement groups.
|
| 97 |
+
bundle_indices (list[list[int]]): bundle indices.
|
| 98 |
+
"""
|
| 99 |
+
|
| 100 |
+
def __init__(
|
| 101 |
+
self,
|
| 102 |
+
config: RolloutConfig,
|
| 103 |
+
model_config: HFModelConfig,
|
| 104 |
+
is_reward_model: bool,
|
| 105 |
+
rollout_mode: RolloutMode,
|
| 106 |
+
workers: list[ActorHandle],
|
| 107 |
+
replica_rank: int,
|
| 108 |
+
max_colocate_count: int,
|
| 109 |
+
pgs: list[PlacementGroup] = None,
|
| 110 |
+
bundle_indices: list[list[int]] = None,
|
| 111 |
+
):
|
| 112 |
+
os.environ["TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL"] = "1"
|
| 113 |
+
assert torch.cuda.is_available(), "TRTLLM http server should run on GPU node"
|
| 114 |
+
|
| 115 |
+
self.config: RolloutConfig = omega_conf_to_dataclass(config)
|
| 116 |
+
self.model_config: HFModelConfig = omega_conf_to_dataclass(model_config, dataclass_type=HFModelConfig)
|
| 117 |
+
self.is_reward_model = is_reward_model
|
| 118 |
+
max_position_embeddings = get_max_position_embeddings(self.model_config.hf_config)
|
| 119 |
+
if self.config.max_model_len is None:
|
| 120 |
+
self.config.max_model_len = max_position_embeddings
|
| 121 |
+
else:
|
| 122 |
+
if self.config.max_model_len > max_position_embeddings:
|
| 123 |
+
raise ValueError(
|
| 124 |
+
f"max_model_len ({self.config.max_model_len}) should be less than or equal to "
|
| 125 |
+
f"max_position_embeddings ({max_position_embeddings})"
|
| 126 |
+
)
|
| 127 |
+
self.rollout_mode = rollout_mode
|
| 128 |
+
self.workers = workers
|
| 129 |
+
self.replica_rank = replica_rank
|
| 130 |
+
self.max_colocate_count = max_colocate_count
|
| 131 |
+
self.pgs = pgs
|
| 132 |
+
self.bundle_indices = bundle_indices
|
| 133 |
+
# model weights version, set by ServerAdapter when update weights.
|
| 134 |
+
self.global_steps = None
|
| 135 |
+
# Set when generation is allowed; cleared during weight sync to block new requests.
|
| 136 |
+
self._generation_allowed = asyncio.Event()
|
| 137 |
+
self._generation_allowed.set()
|
| 138 |
+
# Set by abort_all_requests(reject_request=True) when no resume is coming soon.
|
| 139 |
+
self._rejecting = False
|
| 140 |
+
|
| 141 |
+
self.profiler_controller = self._init_profiler_controller()
|
| 142 |
+
|
| 143 |
+
self.is_vlm_model = (
|
| 144 |
+
self.model_config.hf_config is not None and hasattr(self.model_config.hf_config, "vision_config")
|
| 145 |
+
) or hasattr(self.model_config, "vision_config")
|
| 146 |
+
|
| 147 |
+
# used for http server
|
| 148 |
+
self._server_address = ray.util.get_node_ip_address().strip("[]")
|
| 149 |
+
self._server_port = None
|
| 150 |
+
|
| 151 |
+
logger.info(f"TRTLLMHttpServer, replica_rank: {self.replica_rank}")
|
| 152 |
+
|
| 153 |
+
_end_id, _stop_ids = _resolve_chat_stop_tokens(self.model_config)
|
| 154 |
+
|
| 155 |
+
logger.info(f"TRT-LLM resolved end_id={_end_id}, stop_ids={_stop_ids}")
|
| 156 |
+
|
| 157 |
+
self._use_torch_sampler = bool(int(os.environ.get("TLLM_USE_TORCHSAMPLER", "0")))
|
| 158 |
+
|
| 159 |
+
if self._use_torch_sampler:
|
| 160 |
+
self.sampling_args = {
|
| 161 |
+
"detokenize": True,
|
| 162 |
+
"end_id": _end_id,
|
| 163 |
+
"stop_token_ids": _stop_ids,
|
| 164 |
+
"pad_id": self.model_config.hf_config.pad_token_id,
|
| 165 |
+
"include_stop_str_in_output": True,
|
| 166 |
+
}
|
| 167 |
+
else:
|
| 168 |
+
self.sampling_args = {
|
| 169 |
+
"detokenize": False,
|
| 170 |
+
"end_id": -1,
|
| 171 |
+
"pad_id": self.model_config.hf_config.pad_token_id,
|
| 172 |
+
"stop_token_ids": _stop_ids,
|
| 173 |
+
"include_stop_str_in_output": True,
|
| 174 |
+
}
|
| 175 |
+
logger.info(f"use_torch_sampler={self._use_torch_sampler}, sampling_args={self.sampling_args}")
|
| 176 |
+
|
| 177 |
+
def get_server_address(self):
|
| 178 |
+
"""Get http server address and port."""
|
| 179 |
+
assert self._server_port is not None, "http server is not launched, port is None"
|
| 180 |
+
return self._server_address, self._server_port
|
| 181 |
+
|
| 182 |
+
async def launch_server(self):
|
| 183 |
+
from tensorrt_llm import AsyncLLM
|
| 184 |
+
from tensorrt_llm.llmapi import CapacitySchedulerPolicy, CudaGraphConfig, KvCacheConfig, SchedulerConfig
|
| 185 |
+
|
| 186 |
+
try:
|
| 187 |
+
from tensorrt_llm.llmapi.llm_args import ExecutorMemoryType, SleepConfig
|
| 188 |
+
except ImportError:
|
| 189 |
+
ExecutorMemoryType = None
|
| 190 |
+
SleepConfig = None
|
| 191 |
+
from tensorrt_llm.serve import OpenAIServer
|
| 192 |
+
|
| 193 |
+
assert self.config.pipeline_model_parallel_size == 1, "pipeline_model_parallel_size > 1 is not supported yet"
|
| 194 |
+
|
| 195 |
+
engine_kwargs = self.config.get("engine_kwargs", {}).get("trtllm", {}) or {}
|
| 196 |
+
# Pop kv_cache_config from engine_kwargs to merge into KvCacheConfig constructor,
|
| 197 |
+
# otherwise **engine_kwargs unpacking in llm_kwargs would overwrite the entire
|
| 198 |
+
# KvCacheConfig object, losing free_gpu_memory_fraction and enable_block_reuse.
|
| 199 |
+
kv_cache_overrides = engine_kwargs.pop("kv_cache_config", {})
|
| 200 |
+
kv_cache_kwargs = {
|
| 201 |
+
"enable_block_reuse": self.config.enable_prefix_caching,
|
| 202 |
+
"free_gpu_memory_fraction": self.config.gpu_memory_utilization,
|
| 203 |
+
**kv_cache_overrides,
|
| 204 |
+
}
|
| 205 |
+
kv_cache_config = KvCacheConfig(**kv_cache_kwargs)
|
| 206 |
+
|
| 207 |
+
per_worker_gpu_share = 1.0 / self.max_colocate_count
|
| 208 |
+
|
| 209 |
+
quantization = self.config.quantization
|
| 210 |
+
if quantization is not None:
|
| 211 |
+
if quantization == "fp8":
|
| 212 |
+
FP8_BLOCK_QUANT_KWARGS = {
|
| 213 |
+
"activation_scheme": "dynamic",
|
| 214 |
+
"fmt": "e4m3",
|
| 215 |
+
"quant_method": "fp8",
|
| 216 |
+
"weight_block_size": [128, 128],
|
| 217 |
+
}
|
| 218 |
+
engine_kwargs["model_kwargs"] = {"quantization_config": FP8_BLOCK_QUANT_KWARGS}
|
| 219 |
+
if self.config.load_format != "dummy":
|
| 220 |
+
raise ValueError("FP8 quantization is only supported for dummy load format")
|
| 221 |
+
else:
|
| 222 |
+
raise ValueError(f"Currently only support fp8 quantization, got: {quantization}")
|
| 223 |
+
|
| 224 |
+
llm_kwargs = {
|
| 225 |
+
"model": self.model_config.local_path,
|
| 226 |
+
"backend": "pytorch",
|
| 227 |
+
"dtype": self.config.dtype,
|
| 228 |
+
"enable_chunked_prefill": self.config.enable_chunked_prefill,
|
| 229 |
+
"skip_tokenizer_init": self.config.skip_tokenizer_init,
|
| 230 |
+
"orchestrator_type": "ray",
|
| 231 |
+
"kv_cache_config": kv_cache_config,
|
| 232 |
+
"max_seq_len": self.config.max_model_len,
|
| 233 |
+
"max_batch_size": self.config.max_num_seqs,
|
| 234 |
+
"max_num_tokens": self.config.max_num_batched_tokens,
|
| 235 |
+
"tensor_parallel_size": self.config.tensor_model_parallel_size,
|
| 236 |
+
"pipeline_parallel_size": self.config.pipeline_model_parallel_size,
|
| 237 |
+
"moe_expert_parallel_size": self.config.expert_parallel_size,
|
| 238 |
+
"moe_tensor_parallel_size": self.config.moe_tensor_parallel_size,
|
| 239 |
+
"load_format": self.config.load_format,
|
| 240 |
+
"trust_remote_code": self.model_config.trust_remote_code,
|
| 241 |
+
"placement_groups": self.pgs,
|
| 242 |
+
"placement_bundle_indices": self.bundle_indices,
|
| 243 |
+
"per_worker_gpu_share": per_worker_gpu_share,
|
| 244 |
+
"sleep_config": SleepConfig(
|
| 245 |
+
restore_modes={
|
| 246 |
+
ExecutorMemoryType.MODEL_WEIGHTS_MAIN: "NONE",
|
| 247 |
+
ExecutorMemoryType.KV_CACHE: "NONE",
|
| 248 |
+
}
|
| 249 |
+
)
|
| 250 |
+
if self.config.enable_sleep_mode and SleepConfig is not None
|
| 251 |
+
else None,
|
| 252 |
+
"allreduce_strategy": "NCCL",
|
| 253 |
+
"sampler_type": "TorchSampler" if self._use_torch_sampler else "TRTLLMSampler",
|
| 254 |
+
**engine_kwargs,
|
| 255 |
+
}
|
| 256 |
+
|
| 257 |
+
self_defined_extension = {
|
| 258 |
+
"ray_worker_extension_cls": "verl.workers.rollout.trtllm_rollout.trtllm_worker_extension.WorkerExtension",
|
| 259 |
+
}
|
| 260 |
+
if self.is_vlm_model:
|
| 261 |
+
llm_kwargs.update(self_defined_extension)
|
| 262 |
+
else:
|
| 263 |
+
# TODO: once TRT-LLM WorkerExtension includes wait_for_engine_idle,
|
| 264 |
+
# replace with "tensorrt_llm.llmapi.rlhf_utils.WorkerExtension" directly.
|
| 265 |
+
llm_kwargs.update(
|
| 266 |
+
{
|
| 267 |
+
"ray_worker_extension_cls": (
|
| 268 |
+
"verl.workers.rollout.trtllm_rollout.trtllm_worker_extension.RlhfWorkerExtension"
|
| 269 |
+
),
|
| 270 |
+
}
|
| 271 |
+
)
|
| 272 |
+
|
| 273 |
+
if self.is_reward_model:
|
| 274 |
+
llm_kwargs.update(
|
| 275 |
+
{
|
| 276 |
+
"cuda_graph_config": None,
|
| 277 |
+
"disable_overlap_scheduler": True,
|
| 278 |
+
}
|
| 279 |
+
)
|
| 280 |
+
else:
|
| 281 |
+
llm_kwargs.update(
|
| 282 |
+
{
|
| 283 |
+
"cuda_graph_config": CudaGraphConfig(
|
| 284 |
+
enable_padding=True,
|
| 285 |
+
batch_sizes=self.config.cudagraph_capture_sizes,
|
| 286 |
+
max_batch_size=0 if self.config.cudagraph_capture_sizes else self.config.max_num_seqs,
|
| 287 |
+
),
|
| 288 |
+
"scheduler_config": SchedulerConfig(
|
| 289 |
+
capacity_scheduler_policy=CapacitySchedulerPolicy.MAX_UTILIZATION,
|
| 290 |
+
),
|
| 291 |
+
}
|
| 292 |
+
)
|
| 293 |
+
|
| 294 |
+
self.llm = await AsyncLLM(**llm_kwargs)
|
| 295 |
+
import inspect
|
| 296 |
+
|
| 297 |
+
init_params = inspect.signature(OpenAIServer.__init__).parameters
|
| 298 |
+
if "generator" in init_params:
|
| 299 |
+
trtllm_server = OpenAIServer(
|
| 300 |
+
generator=self.llm,
|
| 301 |
+
model=self.model_config.local_path,
|
| 302 |
+
tool_parser=None,
|
| 303 |
+
server_role=None,
|
| 304 |
+
metadata_server_cfg=None,
|
| 305 |
+
)
|
| 306 |
+
else:
|
| 307 |
+
trtllm_server = OpenAIServer(
|
| 308 |
+
llm=self.llm,
|
| 309 |
+
model=self.model_config.local_path,
|
| 310 |
+
tool_parser=None,
|
| 311 |
+
server_role=None,
|
| 312 |
+
metadata_server_cfg=None,
|
| 313 |
+
)
|
| 314 |
+
|
| 315 |
+
app = trtllm_server.app
|
| 316 |
+
self._server_port, self._server_task = await run_uvicorn(app, None, self._server_address)
|
| 317 |
+
|
| 318 |
+
async def generate(
|
| 319 |
+
self,
|
| 320 |
+
prompt_ids: str | list[int],
|
| 321 |
+
sampling_params: dict[str, Any],
|
| 322 |
+
request_id: str,
|
| 323 |
+
image_data: Optional[list[Any]] = None,
|
| 324 |
+
video_data: Optional[list[Any]] = None,
|
| 325 |
+
audio_data: Optional[list[Any]] = None,
|
| 326 |
+
mm_processor_kwargs: Optional[dict[str, Any]] = None,
|
| 327 |
+
) -> TokenOutput:
|
| 328 |
+
from tensorrt_llm.llmapi import SamplingParams
|
| 329 |
+
|
| 330 |
+
max_tokens = min(
|
| 331 |
+
self.config.response_length,
|
| 332 |
+
self.config.prompt_length + self.config.response_length - len(prompt_ids),
|
| 333 |
+
)
|
| 334 |
+
max_tokens = max(0, min(max_tokens, self.config.max_model_len - len(prompt_ids)))
|
| 335 |
+
sampling_params["max_tokens"] = max_tokens
|
| 336 |
+
# TorchSampler: logprobs=0 means sampled-token logprob; TRTLLMSampler: logprobs=1
|
| 337 |
+
_want_logprobs = sampling_params.pop("logprobs", False)
|
| 338 |
+
if self._use_torch_sampler:
|
| 339 |
+
sampling_params["logprobs"] = 0 if _want_logprobs else None
|
| 340 |
+
else:
|
| 341 |
+
sampling_params["logprobs"] = 1 if _want_logprobs else None
|
| 342 |
+
if sampling_params["top_k"] == -1:
|
| 343 |
+
sampling_params["top_k"] = 0
|
| 344 |
+
sampling_params.update(self.sampling_args)
|
| 345 |
+
|
| 346 |
+
trt_llm_sampling_params = SamplingParams(**sampling_params)
|
| 347 |
+
if audio_data is not None:
|
| 348 |
+
raise NotImplementedError("TRT-LLM rollout does not support audio inputs yet.")
|
| 349 |
+
|
| 350 |
+
rejected = await self._park_until_allowed(request_id)
|
| 351 |
+
if rejected is not None:
|
| 352 |
+
return rejected
|
| 353 |
+
if self.is_vlm_model and (image_data or video_data):
|
| 354 |
+
deduped_ids = qwen2_5_vl_dedup_image_tokens(prompt_ids, self.model_config.processor)
|
| 355 |
+
org_prompt = self.llm.tokenizer.decode(deduped_ids)
|
| 356 |
+
input_dict = {
|
| 357 |
+
"prompt": org_prompt,
|
| 358 |
+
"multi_modal_data": {},
|
| 359 |
+
"mm_processor_kwargs": dict(mm_processor_kwargs or {}),
|
| 360 |
+
}
|
| 361 |
+
if image_data:
|
| 362 |
+
input_dict["multi_modal_data"]["image"] = image_data
|
| 363 |
+
if video_data:
|
| 364 |
+
input_dict["multi_modal_data"]["video"] = video_data
|
| 365 |
+
|
| 366 |
+
outputs = await self.llm.generate_async(
|
| 367 |
+
inputs=input_dict,
|
| 368 |
+
sampling_params=trt_llm_sampling_params,
|
| 369 |
+
)
|
| 370 |
+
else:
|
| 371 |
+
outputs = await self.llm.generate_async(
|
| 372 |
+
inputs=prompt_ids,
|
| 373 |
+
sampling_params=trt_llm_sampling_params,
|
| 374 |
+
)
|
| 375 |
+
token_ids = outputs.outputs[0].token_ids
|
| 376 |
+
log_probs = None
|
| 377 |
+
if outputs.outputs[0].logprobs is not None:
|
| 378 |
+
# When logprobs=1, TRT-LLM returns only the sampled token's logprob at each position.
|
| 379 |
+
# Extract log_probs before checking finish_reason so cancelled (partial) requests also
|
| 380 |
+
# return log_probs for their already-generated tokens.
|
| 381 |
+
log_probs = [list(d.values())[0].logprob for d in outputs.outputs[0].logprobs]
|
| 382 |
+
if outputs.outputs[0].finish_reason == "cancelled":
|
| 383 |
+
return TokenOutput(
|
| 384 |
+
token_ids=token_ids,
|
| 385 |
+
log_probs=log_probs,
|
| 386 |
+
stop_reason="aborted",
|
| 387 |
+
extra_fields={"global_steps": self.global_steps},
|
| 388 |
+
)
|
| 389 |
+
return TokenOutput(token_ids=token_ids, log_probs=log_probs, extra_fields={"global_steps": self.global_steps})
|
| 390 |
+
|
| 391 |
+
async def set_global_steps(self, global_steps: int):
|
| 392 |
+
"""Set the global steps of the model weights."""
|
| 393 |
+
self.global_steps = global_steps
|
| 394 |
+
|
| 395 |
+
async def abort_all_requests(self, reject_request: bool = False):
|
| 396 |
+
"""Abort all in-flight requests and block new ones. Call resume_generation() to unblock.
|
| 397 |
+
|
| 398 |
+
Args:
|
| 399 |
+
reject_request: Fail requests that arrive while generation is blocked instead of
|
| 400 |
+
holding them. Pass True when this server is leaving the load balancer with no
|
| 401 |
+
resume coming soon, so the client re-routes them to an active replica.
|
| 402 |
+
"""
|
| 403 |
+
self._generation_allowed.clear()
|
| 404 |
+
self._rejecting = reject_request
|
| 405 |
+
await self.llm.pause_generation()
|
| 406 |
+
# TODO: remove once TRT-LLM is upgraded to a version where pause_generation()
|
| 407 |
+
# drains internally (https://github.com/NVIDIA/TensorRT-LLM/pull/13784).
|
| 408 |
+
await self.llm.collective_rpc("wait_for_engine_idle")
|
| 409 |
+
if self.config.enable_prefix_caching:
|
| 410 |
+
await self.llm.collective_rpc("reset_prefix_cache")
|
| 411 |
+
|
| 412 |
+
async def resume_generation(self):
|
| 413 |
+
"""Unblock new generation requests after abort_all_requests()."""
|
| 414 |
+
await self.llm.resume_generation()
|
| 415 |
+
self._rejecting = False
|
| 416 |
+
self._generation_allowed.set()
|
| 417 |
+
|
| 418 |
+
async def _park_until_allowed(self, request_id: str) -> Optional[TokenOutput]:
|
| 419 |
+
"""Wait out a blocked gate, or fail the request when the gate rejects late arrivals.
|
| 420 |
+
|
| 421 |
+
Returns None once generation is allowed, or an aborted TokenOutput telling the client
|
| 422 |
+
to re-route. Rejection is checked before waiting, so the request never reaches the engine.
|
| 423 |
+
"""
|
| 424 |
+
while not self._generation_allowed.is_set():
|
| 425 |
+
if self._rejecting:
|
| 426 |
+
logger.debug("rejecting request %s: server left rotation while generation is blocked", request_id)
|
| 427 |
+
return TokenOutput(
|
| 428 |
+
token_ids=[],
|
| 429 |
+
log_probs=None,
|
| 430 |
+
stop_reason="aborted",
|
| 431 |
+
extra_fields={"global_steps": self.global_steps},
|
| 432 |
+
)
|
| 433 |
+
await self._generation_allowed.wait()
|
| 434 |
+
return None
|
| 435 |
+
|
| 436 |
+
async def clear_kv_cache(self):
|
| 437 |
+
"""Invalidate prefix cache entries after weight update."""
|
| 438 |
+
await self.llm.collective_rpc("reset_prefix_cache")
|
| 439 |
+
|
| 440 |
+
async def release_kv_cache(self):
|
| 441 |
+
"""Release only kv_cache GPU memory, keeping model weights intact.
|
| 442 |
+
|
| 443 |
+
This is used during weight sync to free GPU memory for new weights.
|
| 444 |
+
"""
|
| 445 |
+
if not self.config.free_cache_engine:
|
| 446 |
+
return
|
| 447 |
+
await self.llm.release(tags=["kv_cache"])
|
| 448 |
+
|
| 449 |
+
async def resume_kv_cache(self):
|
| 450 |
+
"""Restore kv_cache GPU memory after a weight sync. Counterpart to release_kv_cache()."""
|
| 451 |
+
await self.llm.resume(tags=["kv_cache"])
|
| 452 |
+
|
| 453 |
+
async def wake_up(self):
|
| 454 |
+
from verl.workers.rollout.trtllm_rollout.trtllm_rollout import ServerAdapter
|
| 455 |
+
|
| 456 |
+
if self.rollout_mode == RolloutMode.HYBRID:
|
| 457 |
+
# In hybrid mode, rollout is wake up in `update_weights`
|
| 458 |
+
raise ValueError(f"wake_up not support rollout_mode {self.rollout_mode}")
|
| 459 |
+
if self.rollout_mode == RolloutMode.COLOCATED:
|
| 460 |
+
await self.llm.resume(tags=ServerAdapter.get_full_tags())
|
| 461 |
+
elif self.rollout_mode == RolloutMode.STANDALONE:
|
| 462 |
+
logger.info("skip wake_up in standalone mode")
|
| 463 |
+
|
| 464 |
+
async def sleep(self):
|
| 465 |
+
from verl.workers.rollout.trtllm_rollout.trtllm_rollout import ServerAdapter
|
| 466 |
+
|
| 467 |
+
if not self.config.free_cache_engine:
|
| 468 |
+
return
|
| 469 |
+
|
| 470 |
+
if self.rollout_mode == RolloutMode.HYBRID:
|
| 471 |
+
await self.llm.release(tags=ServerAdapter.get_full_tags())
|
| 472 |
+
elif self.rollout_mode == RolloutMode.COLOCATED:
|
| 473 |
+
await self.llm.release(tags=ServerAdapter.get_full_tags())
|
| 474 |
+
elif self.rollout_mode == RolloutMode.STANDALONE:
|
| 475 |
+
logger.info("skip sleep in standalone mode")
|
| 476 |
+
|
| 477 |
+
async def report_device_ids(self) -> list[str]:
|
| 478 |
+
"""Report GPU device UUIDs from TRT-LLM workers."""
|
| 479 |
+
return await self.llm.collective_rpc(
|
| 480 |
+
"report_device_id",
|
| 481 |
+
unique_reply_rank=0,
|
| 482 |
+
)
|
| 483 |
+
|
| 484 |
+
async def start_profile(self, **kwargs):
|
| 485 |
+
if self.profiler_controller.check_enable() and self.profiler_controller.check_this_rank():
|
| 486 |
+
await self.llm.collective_rpc("start_profile")
|
| 487 |
+
|
| 488 |
+
async def stop_profile(self):
|
| 489 |
+
if self.profiler_controller.check_enable() and self.profiler_controller.check_this_rank():
|
| 490 |
+
await self.llm.collective_rpc("stop_profile")
|
| 491 |
+
# Relocate the engine's traces into save_path (when relocate_results is set) so the
|
| 492 |
+
# training worker's single end-of-run upload of the whole save_path picks them up. The
|
| 493 |
+
# rollout engine does not run the finish command itself: it shares save_path with the
|
| 494 |
+
# colocated training worker, so uploading here too would send the same directory twice.
|
| 495 |
+
relocate_rollout_traces(
|
| 496 |
+
self.profiler_controller.config,
|
| 497 |
+
self.replica_rank,
|
| 498 |
+
self.replica_world_size,
|
| 499 |
+
self.profiler_keep_global_ranks,
|
| 500 |
+
)
|
| 501 |
+
|
| 502 |
+
def _init_profiler_controller(self) -> DistProfiler:
|
| 503 |
+
profiler_config = self.config.profiler
|
| 504 |
+
tool_config = None
|
| 505 |
+
if profiler_config is not None:
|
| 506 |
+
if profiler_config.tool in ["torch", "npu"]:
|
| 507 |
+
tool_config = omega_conf_to_dataclass((profiler_config.tool_config or {}).get(profiler_config.tool))
|
| 508 |
+
elif profiler_config.tool == "nsys":
|
| 509 |
+
# nsys config lives in global_tool_config, not tool_config
|
| 510 |
+
from verl.utils.profiler.config import NsightToolConfig
|
| 511 |
+
|
| 512 |
+
raw = (profiler_config.global_tool_config or {}).get("nsys")
|
| 513 |
+
tool_config = omega_conf_to_dataclass(raw) if raw is not None else NsightToolConfig()
|
| 514 |
+
elif profiler_config.tool is not None:
|
| 515 |
+
logger.warning(f"trtllm rollout: unsupported profiler tool '{profiler_config.tool}', disabling")
|
| 516 |
+
profiler_config = None
|
| 517 |
+
# `ranks` in the rollout profiler config are global GPU ranks (as in the training roles);
|
| 518 |
+
# map them to the replica that owns them so e.g. ranks=[0, 8] with tp=8 profiles the replicas
|
| 519 |
+
# holding global ranks 0 and 8 (replicas 0 and 1), not replica indices 0 and 8.
|
| 520 |
+
self.replica_world_size = (
|
| 521 |
+
self.config.tensor_model_parallel_size
|
| 522 |
+
* self.config.data_parallel_size
|
| 523 |
+
* self.config.pipeline_model_parallel_size
|
| 524 |
+
)
|
| 525 |
+
# A tp>1 engine profiles its whole replica, but the user asked for specific global GPU ranks;
|
| 526 |
+
# keep only those when relocating so ranks=[0, 8] yields exactly GPU 0 and 8, not their tp-mates.
|
| 527 |
+
self.profiler_keep_global_ranks = rollout_profiler_global_ranks(profiler_config)
|
| 528 |
+
return build_rollout_dist_profiler(
|
| 529 |
+
self.replica_rank, self.replica_world_size, config=profiler_config, tool_config=tool_config
|
| 530 |
+
)
|
| 531 |
+
|
| 532 |
+
|
| 533 |
+
class TRTLLMReplica(RolloutReplica):
|
| 534 |
+
def __init__(
|
| 535 |
+
self,
|
| 536 |
+
replica_rank: int,
|
| 537 |
+
config: RolloutConfig,
|
| 538 |
+
model_config: DictConfig,
|
| 539 |
+
gpus_per_node: int = 8,
|
| 540 |
+
is_reward_model: bool = False,
|
| 541 |
+
is_teacher_model: bool = False,
|
| 542 |
+
name_suffix: str = "",
|
| 543 |
+
) -> None:
|
| 544 |
+
if is_teacher_model:
|
| 545 |
+
raise NotImplementedError("TRTLLMReplica doesn't support teacher model yet.")
|
| 546 |
+
super().__init__(
|
| 547 |
+
replica_rank, config, model_config, gpus_per_node, is_reward_model, is_teacher_model, name_suffix
|
| 548 |
+
)
|
| 549 |
+
self.node_ip = ray.util.get_node_ip_address().strip("[]")
|
| 550 |
+
|
| 551 |
+
def rollout_worker_use_gpu(self) -> bool:
|
| 552 |
+
return False
|
| 553 |
+
|
| 554 |
+
def get_pgs_and_bundle_indices(self) -> tuple[list[PlacementGroup], list[list[int]]]:
|
| 555 |
+
"""Get placement groups and bundle indices for the replica."""
|
| 556 |
+
|
| 557 |
+
start_pg_index = 0
|
| 558 |
+
local_bundle_index = 0
|
| 559 |
+
|
| 560 |
+
# For SubRayResourcePool, the replica is assigned sub pool specific for this replica.
|
| 561 |
+
if isinstance(self.resource_pool, SubRayResourcePool):
|
| 562 |
+
assert self.resource_pool.subgroup_world_size == self.world_size, (
|
| 563 |
+
"Subgroup world size must be equal to world size"
|
| 564 |
+
)
|
| 565 |
+
local_bundle_index = self.resource_pool.start_bundle_index
|
| 566 |
+
# For RayResourcePool, the replica is assigned to entire resource pool.
|
| 567 |
+
# We need to find start pg index and local bundle index based on replica rank.
|
| 568 |
+
else:
|
| 569 |
+
# In standalone mode, init_standalone() creates a per-replica RayResourcePool
|
| 570 |
+
# that contains only world_size bundles for this replica. Start at bundle 0.
|
| 571 |
+
# In colocated/hybrid mode, the shared pool spans all replicas, so offset by rank.
|
| 572 |
+
if self.rollout_mode == RolloutMode.STANDALONE:
|
| 573 |
+
local_bundle_index = 0
|
| 574 |
+
else:
|
| 575 |
+
local_bundle_index = self.world_size * self.replica_rank
|
| 576 |
+
|
| 577 |
+
while (
|
| 578 |
+
start_pg_index < len(self.resource_pool.pgs)
|
| 579 |
+
and local_bundle_index >= self.resource_pool.pgs[start_pg_index].bundle_count
|
| 580 |
+
):
|
| 581 |
+
local_bundle_index -= self.resource_pool.pgs[start_pg_index].bundle_count
|
| 582 |
+
start_pg_index += 1
|
| 583 |
+
assert (
|
| 584 |
+
start_pg_index < len(self.resource_pool.pgs)
|
| 585 |
+
and local_bundle_index < self.resource_pool.pgs[start_pg_index].bundle_count
|
| 586 |
+
), "Start pg index or local bundle index out of range"
|
| 587 |
+
|
| 588 |
+
# Global Bundle View for Replica x 2 & TP=4:
|
| 589 |
+
# ┌───────────────────┬───────────────────┐
|
| 590 |
+
# │ Placement Group 0 │ Placement Group 1 │
|
| 591 |
+
# ├────┬────┬────┬────┼────┬────┬────┬────┤
|
| 592 |
+
# │ 0 │ 1 │ 2 │ 3 │ 0 │ 1 │ 2 │ 3 │
|
| 593 |
+
# └────┴────┴────┴────┴────┴────┴────┴────┘
|
| 594 |
+
# └───────────────┘ └───────────────┘
|
| 595 |
+
# Replica 0 Replica 1
|
| 596 |
+
# (4 GPUs) (4 GPUs)
|
| 597 |
+
|
| 598 |
+
left_bundle_count = self.world_size
|
| 599 |
+
|
| 600 |
+
pgs = []
|
| 601 |
+
bundle_indices = []
|
| 602 |
+
|
| 603 |
+
for pg in self.resource_pool.pgs[start_pg_index:]:
|
| 604 |
+
if left_bundle_count == 0:
|
| 605 |
+
break
|
| 606 |
+
|
| 607 |
+
left_bundle_count_in_pg = min(left_bundle_count, pg.bundle_count - local_bundle_index)
|
| 608 |
+
pg_bundle_indices = [local_bundle_index + idx for idx in range(left_bundle_count_in_pg)]
|
| 609 |
+
pgs.append(pg)
|
| 610 |
+
bundle_indices.append(pg_bundle_indices)
|
| 611 |
+
left_bundle_count -= left_bundle_count_in_pg
|
| 612 |
+
local_bundle_index = 0
|
| 613 |
+
|
| 614 |
+
assert left_bundle_count == 0, "all bundle indices should be assigned"
|
| 615 |
+
|
| 616 |
+
return pgs, bundle_indices
|
| 617 |
+
|
| 618 |
+
async def launch_servers(self):
|
| 619 |
+
assert self.resource_pool.pgs is not None, "placement groups are not initialized"
|
| 620 |
+
|
| 621 |
+
pgs, bundle_indices = self.get_pgs_and_bundle_indices()
|
| 622 |
+
|
| 623 |
+
# Check server process should be launched on the same node as first bundle of first pg.
|
| 624 |
+
first_pg_data = placement_group_table(pgs[0])
|
| 625 |
+
node_id = first_pg_data["bundles_to_node_id"][bundle_indices[0][0]]
|
| 626 |
+
print(f"TRTLLMReplica: {self.replica_rank}")
|
| 627 |
+
print(f"pg node_id: {node_id}")
|
| 628 |
+
print(f"pgs: {pgs}")
|
| 629 |
+
print(f"bundle_indices: {bundle_indices}")
|
| 630 |
+
|
| 631 |
+
# TRTLLMReplica is a 1:1 map from replica to TRTLLMHttpServer.
|
| 632 |
+
name = (
|
| 633 |
+
f"trtllm_server_{self.replica_rank}{self.name_suffix}"
|
| 634 |
+
if not self.is_reward_model
|
| 635 |
+
else f"trtllm_server_reward_{self.replica_rank}{self.name_suffix}"
|
| 636 |
+
)
|
| 637 |
+
_server_env_vars = {var: "1" for var in get_platform().ray_noset_envvars()}
|
| 638 |
+
_server_env_vars.update(get_platform().rollout_env_vars())
|
| 639 |
+
# Propagate profiling env vars to the Ray actor so that RayExecutor
|
| 640 |
+
# (instantiated inside TRTLLMHttpServer) picks them up for inner workers.
|
| 641 |
+
for _prof_var in (
|
| 642 |
+
"TLLM_ENABLE_NSYS",
|
| 643 |
+
"TLLM_NSYS_OUTPUT_DIR",
|
| 644 |
+
"TLLM_USE_TORCHSAMPLER",
|
| 645 |
+
):
|
| 646 |
+
if _val := os.environ.get(_prof_var):
|
| 647 |
+
_server_env_vars[_prof_var] = _val
|
| 648 |
+
server = TRTLLMHttpServer.options(
|
| 649 |
+
scheduling_strategy=ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy(
|
| 650 |
+
node_id=node_id,
|
| 651 |
+
soft=False,
|
| 652 |
+
),
|
| 653 |
+
runtime_env={"env_vars": _server_env_vars},
|
| 654 |
+
name=name,
|
| 655 |
+
max_concurrency=self.max_concurrency,
|
| 656 |
+
).remote(
|
| 657 |
+
config=self.config,
|
| 658 |
+
model_config=self.model_config,
|
| 659 |
+
is_reward_model=self.is_reward_model,
|
| 660 |
+
rollout_mode=self.rollout_mode,
|
| 661 |
+
workers=self.workers,
|
| 662 |
+
replica_rank=self.replica_rank,
|
| 663 |
+
max_colocate_count=self.resource_pool.max_colocate_count,
|
| 664 |
+
pgs=pgs,
|
| 665 |
+
bundle_indices=bundle_indices,
|
| 666 |
+
)
|
| 667 |
+
self.servers.append(server)
|
| 668 |
+
|
| 669 |
+
# launch http server in each node
|
| 670 |
+
await asyncio.gather(*[server.launch_server.remote() for server in self.servers])
|
| 671 |
+
|
| 672 |
+
# get http server address from first server
|
| 673 |
+
server_address, server_port = await self.servers[0].get_server_address.remote()
|
| 674 |
+
self._server_handle = self.servers[0]
|
| 675 |
+
self._server_address = (
|
| 676 |
+
f"[{server_address}]:{server_port}"
|
| 677 |
+
if is_valid_ipv6_address(server_address)
|
| 678 |
+
else f"{server_address}:{server_port}"
|
| 679 |
+
)
|
repro/vendor/verl/workers/rollout/trtllm_rollout/trtllm_rollout.py
ADDED
|
@@ -0,0 +1,544 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 Bytedance Ltd. and/or its affiliates
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import asyncio
|
| 17 |
+
import base64
|
| 18 |
+
import contextlib
|
| 19 |
+
import gc
|
| 20 |
+
import logging
|
| 21 |
+
import os
|
| 22 |
+
import pickle
|
| 23 |
+
import threading
|
| 24 |
+
from contextlib import asynccontextmanager
|
| 25 |
+
from typing import Any, AsyncGenerator, Optional
|
| 26 |
+
|
| 27 |
+
import aiohttp
|
| 28 |
+
import pynvml
|
| 29 |
+
import ray
|
| 30 |
+
import torch
|
| 31 |
+
import torch.distributed as dist
|
| 32 |
+
|
| 33 |
+
try:
|
| 34 |
+
from tensorrt_llm.llmapi.llm_args import ExecutorMemoryType
|
| 35 |
+
except (ImportError, RuntimeError):
|
| 36 |
+
# RuntimeError: FlashInfer's check_cuda_arch() crashes on CPU-only actors
|
| 37 |
+
ExecutorMemoryType = None
|
| 38 |
+
from torch.distributed.device_mesh import DeviceMesh, init_device_mesh
|
| 39 |
+
from torch.multiprocessing.reductions import reduce_tensor
|
| 40 |
+
|
| 41 |
+
from verl.utils.device import get_torch_device
|
| 42 |
+
from verl.utils.net_utils import is_valid_ipv6_address
|
| 43 |
+
from verl.workers.config import HFModelConfig, RolloutConfig
|
| 44 |
+
from verl.workers.rollout.base import BaseRollout
|
| 45 |
+
from verl.workers.rollout.utils import ensure_async_iterator
|
| 46 |
+
|
| 47 |
+
logger = logging.getLogger(__file__)
|
| 48 |
+
logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
|
| 49 |
+
|
| 50 |
+
# Default configuration constants
|
| 51 |
+
DEFAULT_TIMEOUT = 60.0
|
| 52 |
+
DEFAULT_MAX_ATTEMPTS = 3
|
| 53 |
+
DEFAULT_RETRY_DELAY = 2.0
|
| 54 |
+
DEFAULT_MAX_CONNECTIONS = 2000
|
| 55 |
+
DEFAULT_MAX_WAIT_TIME = 300.0
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
@contextlib.contextmanager
|
| 59 |
+
def nvml_context():
|
| 60 |
+
"""Context manager for NVML initialization and shutdown.
|
| 61 |
+
|
| 62 |
+
Raises:
|
| 63 |
+
RuntimeError: If NVML initialization fails
|
| 64 |
+
"""
|
| 65 |
+
try:
|
| 66 |
+
pynvml.nvmlInit()
|
| 67 |
+
yield
|
| 68 |
+
except pynvml.NVMLError as e:
|
| 69 |
+
raise RuntimeError(f"Failed to initialize NVML: {e}") from e
|
| 70 |
+
finally:
|
| 71 |
+
try:
|
| 72 |
+
pynvml.nvmlShutdown()
|
| 73 |
+
except pynvml.NVMLError:
|
| 74 |
+
pass
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
_NVML_INITIALIZED = False
|
| 78 |
+
_NVML_LOCK = threading.Lock()
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def get_device_uuid(id: str | int) -> str:
|
| 82 |
+
"""Get the UUID of a CUDA device using NVML."""
|
| 83 |
+
id = int(id) # pynvml expects int; ray.get_gpu_ids() may return str
|
| 84 |
+
global _NVML_INITIALIZED
|
| 85 |
+
with _NVML_LOCK:
|
| 86 |
+
if not _NVML_INITIALIZED:
|
| 87 |
+
try:
|
| 88 |
+
pynvml.nvmlInit()
|
| 89 |
+
_NVML_INITIALIZED = True
|
| 90 |
+
except pynvml.NVMLError as e:
|
| 91 |
+
raise RuntimeError(f"Failed to initialize NVML: {e}") from e
|
| 92 |
+
|
| 93 |
+
# Get the device handle and UUID
|
| 94 |
+
try:
|
| 95 |
+
handle = pynvml.nvmlDeviceGetHandleByIndex(id)
|
| 96 |
+
uuid = pynvml.nvmlDeviceGetUUID(handle)
|
| 97 |
+
# Ensure the UUID is returned as a string, not bytes
|
| 98 |
+
if isinstance(uuid, bytes):
|
| 99 |
+
return uuid.decode("utf-8")
|
| 100 |
+
elif isinstance(uuid, str):
|
| 101 |
+
return uuid
|
| 102 |
+
else:
|
| 103 |
+
raise RuntimeError(f"Unexpected UUID type: {type(uuid)} for device {id} (global index: {id})")
|
| 104 |
+
except pynvml.NVMLError as e:
|
| 105 |
+
raise RuntimeError(f"Failed to get device UUID for device {id} (global index: {id}): {e}") from e
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
async def _read_async_response(resp: aiohttp.ClientResponse) -> dict[str, Any]:
|
| 109 |
+
if resp.status == 204 or (resp.content_length == 0):
|
| 110 |
+
return {}
|
| 111 |
+
|
| 112 |
+
try:
|
| 113 |
+
return await resp.json(content_type=None)
|
| 114 |
+
except Exception:
|
| 115 |
+
try:
|
| 116 |
+
text = await resp.text()
|
| 117 |
+
except Exception:
|
| 118 |
+
return {}
|
| 119 |
+
return {
|
| 120 |
+
"content_type": (resp.headers.get("Content-Type") or ""),
|
| 121 |
+
"text": text,
|
| 122 |
+
}
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
class AsyncTRTLLMHttpAdapter:
|
| 126 |
+
def __init__(
|
| 127 |
+
self,
|
| 128 |
+
host: str,
|
| 129 |
+
port: int,
|
| 130 |
+
timeout: float = DEFAULT_TIMEOUT,
|
| 131 |
+
max_attempts: int = DEFAULT_MAX_ATTEMPTS,
|
| 132 |
+
retry_delay: float = DEFAULT_RETRY_DELAY,
|
| 133 |
+
max_connections: int = DEFAULT_MAX_CONNECTIONS,
|
| 134 |
+
):
|
| 135 |
+
self.host = host
|
| 136 |
+
self.port = port
|
| 137 |
+
self.timeout = timeout
|
| 138 |
+
self.max_attempts = max_attempts
|
| 139 |
+
self.retry_delay = retry_delay
|
| 140 |
+
self.max_connections = max_connections
|
| 141 |
+
|
| 142 |
+
@asynccontextmanager
|
| 143 |
+
async def _get_session(self) -> aiohttp.ClientSession:
|
| 144 |
+
"""Context manager for safe session access with proper connection pooling.
|
| 145 |
+
|
| 146 |
+
Yields:
|
| 147 |
+
aiohttp.ClientSession: Session instance for making HTTP requests
|
| 148 |
+
|
| 149 |
+
Note:
|
| 150 |
+
This method creates a new session for each request to avoid resource competition
|
| 151 |
+
while still maintaining proper connection pooling through the shared connector.
|
| 152 |
+
"""
|
| 153 |
+
# Create a new session for each request to avoid resource competition
|
| 154 |
+
connector = aiohttp.TCPConnector(
|
| 155 |
+
limit=self.max_connections,
|
| 156 |
+
limit_per_host=self.max_connections // 4,
|
| 157 |
+
ttl_dns_cache=300,
|
| 158 |
+
use_dns_cache=True,
|
| 159 |
+
)
|
| 160 |
+
timeout = aiohttp.ClientTimeout(total=self.timeout)
|
| 161 |
+
session = aiohttp.ClientSession(connector=connector, timeout=timeout)
|
| 162 |
+
|
| 163 |
+
try:
|
| 164 |
+
yield session
|
| 165 |
+
finally:
|
| 166 |
+
# Always close the session to free up resources
|
| 167 |
+
if not session.closed:
|
| 168 |
+
await session.close()
|
| 169 |
+
|
| 170 |
+
async def _make_async_request(
|
| 171 |
+
self,
|
| 172 |
+
endpoint: str,
|
| 173 |
+
payload: Optional[dict[str, Any]] = None,
|
| 174 |
+
timeout: float = DEFAULT_TIMEOUT,
|
| 175 |
+
method: str = "POST",
|
| 176 |
+
return_status: bool = False,
|
| 177 |
+
) -> dict[str, Any] | int:
|
| 178 |
+
"""Make an async HTTP request with retry logic and consistent error handling.
|
| 179 |
+
|
| 180 |
+
Args:
|
| 181 |
+
endpoint (str): The API endpoint to call (without leading slash)
|
| 182 |
+
payload (Optional[Dict[str, Any]], optional): The JSON payload to send.
|
| 183 |
+
Defaults to empty dict if None.
|
| 184 |
+
method (str, optional): HTTP method to use. Defaults to "POST".
|
| 185 |
+
|
| 186 |
+
Returns:
|
| 187 |
+
Dict[str, Any]: The JSON response from the server
|
| 188 |
+
|
| 189 |
+
Raises:
|
| 190 |
+
aiohttp.ClientResponseError: If the HTTP request fails with a client/server error
|
| 191 |
+
RuntimeError: If all retry attempts are exhausted
|
| 192 |
+
|
| 193 |
+
Note:
|
| 194 |
+
- Uses exponential backoff for retries
|
| 195 |
+
- Logs warnings for timeout and connection errors, errors for HTTP errors
|
| 196 |
+
"""
|
| 197 |
+
|
| 198 |
+
url = f"http://{self.host}:{self.port}/{endpoint}"
|
| 199 |
+
|
| 200 |
+
for attempt in range(self.max_attempts):
|
| 201 |
+
try:
|
| 202 |
+
async with self._get_session() as session:
|
| 203 |
+
if method.upper() == "GET":
|
| 204 |
+
async with session.get(url, timeout=timeout) as response:
|
| 205 |
+
response.raise_for_status()
|
| 206 |
+
return response.status if return_status else await _read_async_response(response)
|
| 207 |
+
else:
|
| 208 |
+
async with session.post(url, json=payload or {}, timeout=timeout) as response:
|
| 209 |
+
response.raise_for_status()
|
| 210 |
+
return response.status if return_status else await _read_async_response(response)
|
| 211 |
+
|
| 212 |
+
except asyncio.TimeoutError:
|
| 213 |
+
logger.warning(f"Async request to {endpoint} timed out (attempt {attempt + 1})")
|
| 214 |
+
except aiohttp.ClientConnectorError:
|
| 215 |
+
logger.warning(f"Connection error for {endpoint} (attempt {attempt + 1})")
|
| 216 |
+
except aiohttp.ClientResponseError as e:
|
| 217 |
+
logger.error(f"HTTP error for {endpoint}: {e}")
|
| 218 |
+
raise
|
| 219 |
+
except Exception as e:
|
| 220 |
+
logger.error(f"Unexpected error for {endpoint}: {e}")
|
| 221 |
+
if attempt == self.max_attempts - 1:
|
| 222 |
+
raise
|
| 223 |
+
|
| 224 |
+
if attempt < self.max_attempts - 1:
|
| 225 |
+
await asyncio.sleep(self.retry_delay * (2**attempt))
|
| 226 |
+
|
| 227 |
+
raise RuntimeError(f"Failed to complete async request to {endpoint} after {self.max_attempts} attempts")
|
| 228 |
+
|
| 229 |
+
async def resume_memory_occupation(self, tags: list[str]):
|
| 230 |
+
"""Resume GPU memory occupation (async version).
|
| 231 |
+
|
| 232 |
+
Similar to AsyncEngine, this method handles first-time weight reloading
|
| 233 |
+
by calling release_memory_occupation if needed.
|
| 234 |
+
|
| 235 |
+
Args:
|
| 236 |
+
tags (Optional[List[str]], optional): List of tags to specify which memory to resume.
|
| 237 |
+
If None, resumes all memory. Defaults to None. ["weights", "kv_cache"]
|
| 238 |
+
|
| 239 |
+
Returns:
|
| 240 |
+
Dict[str, Any]: Server response indicating memory resume status
|
| 241 |
+
"""
|
| 242 |
+
return await self._make_async_request("resume_memory", {"tags": tags})
|
| 243 |
+
|
| 244 |
+
async def release_memory_occupation(self, tags: list[str]):
|
| 245 |
+
"""Release GPU memory occupation temporarily (async version).
|
| 246 |
+
|
| 247 |
+
Args:
|
| 248 |
+
tags (Optional[List[str]], optional): List of tags to specify which memory to release.
|
| 249 |
+
If None, releases all memory. Defaults to None. ["weights", "kv_cache"]
|
| 250 |
+
|
| 251 |
+
Returns:
|
| 252 |
+
Dict[str, Any]: Server response indicating memory release status
|
| 253 |
+
"""
|
| 254 |
+
return await self._make_async_request("release_memory", {"tags": tags})
|
| 255 |
+
|
| 256 |
+
async def update_weights(self, weights: dict[str, str]):
|
| 257 |
+
"""Update model weights from tensor data asynchronously.
|
| 258 |
+
|
| 259 |
+
Args:
|
| 260 |
+
weights: A dictionary that maps the device uuid of the weight handles.
|
| 261 |
+
|
| 262 |
+
Returns:
|
| 263 |
+
Dict[str, Any]: Server response containing update status
|
| 264 |
+
"""
|
| 265 |
+
return await self._make_async_request("update_weights", {"weights": weights})
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
class ServerAdapter(BaseRollout):
|
| 269 |
+
# All releasable/resumable weight tags: every ExecutorMemoryType except kv_cache
|
| 270 |
+
# (handled separately) and internal tags prefixed with "_".
|
| 271 |
+
# Fallback to hard-coded list for trtllm versions that don't export ExecutorMemoryType.
|
| 272 |
+
_WEIGHTS_TAGS = (
|
| 273 |
+
[t.value for t in ExecutorMemoryType if t is not ExecutorMemoryType.KV_CACHE and not t.value.startswith("_")]
|
| 274 |
+
if ExecutorMemoryType is not None
|
| 275 |
+
else [
|
| 276 |
+
"sampler",
|
| 277 |
+
"drafter",
|
| 278 |
+
"guided_decoder",
|
| 279 |
+
"spec_resource_manager",
|
| 280 |
+
"model_extra",
|
| 281 |
+
"executor_extra",
|
| 282 |
+
"model",
|
| 283 |
+
"draft_model",
|
| 284 |
+
]
|
| 285 |
+
)
|
| 286 |
+
|
| 287 |
+
@staticmethod
|
| 288 |
+
def get_full_tags() -> list[str]:
|
| 289 |
+
return ServerAdapter._WEIGHTS_TAGS + ["kv_cache"]
|
| 290 |
+
|
| 291 |
+
def __init__(
|
| 292 |
+
self, config: RolloutConfig, model_config: HFModelConfig, device_mesh: DeviceMesh, replica_rank: int = -1
|
| 293 |
+
):
|
| 294 |
+
super().__init__(config, model_config, device_mesh)
|
| 295 |
+
if self.config.quantization == "fp8" and self.model_config.hf_config is not None:
|
| 296 |
+
FP8_BLOCK_QUANT_KWARGS = {
|
| 297 |
+
"activation_scheme": "dynamic",
|
| 298 |
+
"fmt": "e4m3",
|
| 299 |
+
"quant_method": "fp8",
|
| 300 |
+
"weight_block_size": [128, 128],
|
| 301 |
+
}
|
| 302 |
+
self.model_config.hf_config.quantization_config = dict(FP8_BLOCK_QUANT_KWARGS)
|
| 303 |
+
self._adapter = None
|
| 304 |
+
self.hybrid_device_mesh = None
|
| 305 |
+
self.gpu_id = None
|
| 306 |
+
self.is_leader_rank = None
|
| 307 |
+
self.replica_rank = None
|
| 308 |
+
self.is_dp_rank = None
|
| 309 |
+
self._supports_partial_loading = None
|
| 310 |
+
|
| 311 |
+
# hybrid mode
|
| 312 |
+
if self.device_mesh is not None:
|
| 313 |
+
assert device_mesh.mesh_dim_names.index("dp") == 0, "DP dim should always be the first dimension"
|
| 314 |
+
|
| 315 |
+
# Clone a new device mesh for CPU backend only (used for internal ranks communication)
|
| 316 |
+
device_mesh_kwargs = dict(
|
| 317 |
+
mesh_shape=device_mesh.mesh.shape,
|
| 318 |
+
mesh_dim_names=device_mesh.mesh_dim_names,
|
| 319 |
+
)
|
| 320 |
+
self.hybrid_device_mesh = init_device_mesh("cpu", **device_mesh_kwargs)
|
| 321 |
+
|
| 322 |
+
self.hybrid_device_mesh[self.hybrid_device_mesh.mesh_dim_names[1:]]._flatten(mesh_dim_name="exclude_dp")
|
| 323 |
+
self.is_leader_rank = self.hybrid_device_mesh["exclude_dp"].get_local_rank() == 0
|
| 324 |
+
logger.info(f"is_dp_leader: {self.is_leader_rank}")
|
| 325 |
+
logger.info(f"exclude_dp_rank = {self.hybrid_device_mesh['exclude_dp'].get_local_rank()}")
|
| 326 |
+
logger.info(f"exclude_dp_size = {self.hybrid_device_mesh['exclude_dp'].size()}")
|
| 327 |
+
self.gpu_id = ray.get_gpu_ids()[0]
|
| 328 |
+
self.replica_rank = self.hybrid_device_mesh["dp"].get_local_rank()
|
| 329 |
+
assert len(ray.get_gpu_ids()) == 1, "ServerAdapter should run on a single GPU node"
|
| 330 |
+
else:
|
| 331 |
+
rank = int(os.environ["RANK"])
|
| 332 |
+
self.replica_rank = replica_rank
|
| 333 |
+
self.is_leader_rank = rank == 0
|
| 334 |
+
# Required for CUDA IPC handle creation during weight sync for Async RL.
|
| 335 |
+
# Reward/ref models skip weight sync so this can be None.
|
| 336 |
+
self.gpu_id = ray.get_gpu_ids()[0]
|
| 337 |
+
|
| 338 |
+
# Below is required for all modes.
|
| 339 |
+
assert self.replica_rank >= 0, "replica_rank is not set"
|
| 340 |
+
assert self.is_leader_rank is not None, "is_leader_rank is not set"
|
| 341 |
+
|
| 342 |
+
self.node_ip = ray.util.get_node_ip_address().strip("[]")
|
| 343 |
+
|
| 344 |
+
async def get_supports_partial_loading(self) -> bool:
|
| 345 |
+
"""Query and cache whether the model supports partial weight loading."""
|
| 346 |
+
if self._supports_partial_loading is not None:
|
| 347 |
+
return self._supports_partial_loading
|
| 348 |
+
|
| 349 |
+
await self._init_server_adapter()
|
| 350 |
+
try:
|
| 351 |
+
self._supports_partial_loading = await self.server_actor.supports_partial_loading.remote()
|
| 352 |
+
except Exception as e:
|
| 353 |
+
logger.warning(f"Failed to query partial loading support: {e}, defaulting to False")
|
| 354 |
+
self._supports_partial_loading = False
|
| 355 |
+
|
| 356 |
+
logger.info(f"Model supports partial loading: {self._supports_partial_loading}")
|
| 357 |
+
return self._supports_partial_loading
|
| 358 |
+
|
| 359 |
+
async def _init_server_adapter(self):
|
| 360 |
+
if self._adapter is not None:
|
| 361 |
+
return
|
| 362 |
+
|
| 363 |
+
# Standalone mode: lazily build the CPU device mesh from the gloo process group
|
| 364 |
+
# (initialized by initialize_global_process_group_ray before ServerAdapter construction).
|
| 365 |
+
# Reward/ref models that never call resume(), release(), or update_weights() will never build the mesh.
|
| 366 |
+
if self.hybrid_device_mesh is None and self.device_mesh is None:
|
| 367 |
+
assert dist.is_initialized(), "gloo process group must be initialized before building device mesh"
|
| 368 |
+
infer_tp = self.config.tensor_model_parallel_size
|
| 369 |
+
infer_pp = getattr(self.config, "pipeline_model_parallel_size", 1)
|
| 370 |
+
world_size = dist.get_world_size()
|
| 371 |
+
dp = world_size // (infer_tp * infer_pp)
|
| 372 |
+
self.hybrid_device_mesh = init_device_mesh(
|
| 373 |
+
"cpu", mesh_shape=(dp, infer_tp, infer_pp), mesh_dim_names=["dp", "infer_tp", "infer_pp"]
|
| 374 |
+
)
|
| 375 |
+
self.hybrid_device_mesh[self.hybrid_device_mesh.mesh_dim_names[1:]]._flatten(mesh_dim_name="exclude_dp")
|
| 376 |
+
self.is_leader_rank = self.hybrid_device_mesh["exclude_dp"].get_local_rank() == 0
|
| 377 |
+
|
| 378 |
+
# Lazy init http server adapter because http server is launched after hybrid engine.
|
| 379 |
+
self.server_actor = ray.get_actor(f"trtllm_server_{self.replica_rank}")
|
| 380 |
+
server_address, server_port = await self.server_actor.get_server_address.remote()
|
| 381 |
+
assert server_address == self.node_ip, f"server address: {server_address} != node_ip: {self.node_ip}"
|
| 382 |
+
|
| 383 |
+
logger.debug(f"replica_rank={self.replica_rank}, server address: {server_address}, port: {server_port}")
|
| 384 |
+
host = f"[{server_address}]" if is_valid_ipv6_address(server_address) else server_address
|
| 385 |
+
self._adapter = AsyncTRTLLMHttpAdapter(
|
| 386 |
+
host=host,
|
| 387 |
+
port=server_port,
|
| 388 |
+
timeout=self.config.server.timeout,
|
| 389 |
+
max_attempts=self.config.server.max_attempts,
|
| 390 |
+
retry_delay=self.config.server.retry_delay,
|
| 391 |
+
max_connections=self.config.server.max_connections,
|
| 392 |
+
)
|
| 393 |
+
|
| 394 |
+
async def resume(self, tags: list[str]):
|
| 395 |
+
"""Resume rollout weights or kv cache in GPU memory.
|
| 396 |
+
|
| 397 |
+
Args:
|
| 398 |
+
tag: weights or kv_cache.
|
| 399 |
+
"""
|
| 400 |
+
# Synchronize all ranks before resuming KV cache to ensure non-leader ranks
|
| 401 |
+
# have completed actor offloading to CPU, preventing OOM issue.
|
| 402 |
+
if "kv_cache" in tags and self.config.free_cache_engine:
|
| 403 |
+
group = self.hybrid_device_mesh["exclude_dp"].get_group() if self.hybrid_device_mesh is not None else None
|
| 404 |
+
await asyncio.to_thread(dist.barrier, group=group)
|
| 405 |
+
if self.is_leader_rank and self.config.free_cache_engine:
|
| 406 |
+
if "weights" in tags:
|
| 407 |
+
tags = self._WEIGHTS_TAGS
|
| 408 |
+
elif "kv_cache" in tags:
|
| 409 |
+
tags = ["kv_cache"]
|
| 410 |
+
else:
|
| 411 |
+
raise ValueError(f"Invalid tag: {tags}")
|
| 412 |
+
await self._init_server_adapter()
|
| 413 |
+
await self._adapter.resume_memory_occupation(tags=tags)
|
| 414 |
+
|
| 415 |
+
async def release(self):
|
| 416 |
+
"""Release weights and kv cache in GPU memory."""
|
| 417 |
+
if self.is_leader_rank and self.config.free_cache_engine:
|
| 418 |
+
await self._init_server_adapter()
|
| 419 |
+
tags = self._WEIGHTS_TAGS + ["kv_cache"]
|
| 420 |
+
await self._adapter.release_memory_occupation(tags=tags)
|
| 421 |
+
|
| 422 |
+
async def update_weights_from_ipc_handles(self, device_handles):
|
| 423 |
+
"""Update weights from IPC handles."""
|
| 424 |
+
if self.hybrid_device_mesh is not None:
|
| 425 |
+
world_size = self.hybrid_device_mesh["exclude_dp"].size()
|
| 426 |
+
group = self.hybrid_device_mesh["exclude_dp"].get_group()
|
| 427 |
+
else:
|
| 428 |
+
world_size = dist.get_world_size()
|
| 429 |
+
group = None
|
| 430 |
+
|
| 431 |
+
if self.is_leader_rank:
|
| 432 |
+
gathered_handles = [None for _ in range(world_size)]
|
| 433 |
+
else:
|
| 434 |
+
gathered_handles = None
|
| 435 |
+
|
| 436 |
+
await asyncio.to_thread(
|
| 437 |
+
dist.gather_object,
|
| 438 |
+
obj=device_handles,
|
| 439 |
+
object_gather_list=gathered_handles,
|
| 440 |
+
group_dst=0,
|
| 441 |
+
group=group,
|
| 442 |
+
)
|
| 443 |
+
|
| 444 |
+
if self.is_leader_rank:
|
| 445 |
+
all_handles = {k: v for d in gathered_handles for k, v in d.items()}
|
| 446 |
+
await self._adapter.update_weights(all_handles)
|
| 447 |
+
|
| 448 |
+
await asyncio.to_thread(dist.barrier, group=group)
|
| 449 |
+
|
| 450 |
+
async def update_weights(
|
| 451 |
+
self,
|
| 452 |
+
weights: AsyncGenerator[tuple[str, torch.Tensor], None],
|
| 453 |
+
global_steps: int = None,
|
| 454 |
+
wire_format: str = "named_tensors",
|
| 455 |
+
**kwargs,
|
| 456 |
+
):
|
| 457 |
+
"""Update the weights of the rollout model.
|
| 458 |
+
|
| 459 |
+
Args:
|
| 460 |
+
weights: A generator that yields the name of the weight tensor and the tensor itself.
|
| 461 |
+
"""
|
| 462 |
+
assert wire_format == "named_tensors", (
|
| 463 |
+
f"TensorRT-LLM rollout only consumes full named tensors; got wire_format={wire_format!r}"
|
| 464 |
+
)
|
| 465 |
+
if self.is_leader_rank:
|
| 466 |
+
await self._init_server_adapter()
|
| 467 |
+
|
| 468 |
+
total_available_bytes = int(self.config.checkpoint_engine.update_weights_bucket_megabytes) * 1024 * 1024
|
| 469 |
+
|
| 470 |
+
if self.config.get("quantization", None) == "fp8":
|
| 471 |
+
from verl.utils.trtllm.trtllm_fp8_utils import TRTLLMFP8QuantizerHelper
|
| 472 |
+
|
| 473 |
+
fp8_quantizer_helper = TRTLLMFP8QuantizerHelper(self.model_config.hf_config.quantization_config)
|
| 474 |
+
weights = fp8_quantizer_helper.quant_weights_by_name(
|
| 475 |
+
weights,
|
| 476 |
+
dtype=self.model_config.hf_config.dtype,
|
| 477 |
+
)
|
| 478 |
+
|
| 479 |
+
try:
|
| 480 |
+
device_uuid = get_device_uuid(int(self.gpu_id))
|
| 481 |
+
except Exception as e:
|
| 482 |
+
logger.error(f"Failed to get device UUID in update_weights(): {e}")
|
| 483 |
+
device_uuid = None
|
| 484 |
+
raise e
|
| 485 |
+
|
| 486 |
+
cur_available_bytes = total_available_bytes
|
| 487 |
+
cur_handles = []
|
| 488 |
+
|
| 489 |
+
async def flush():
|
| 490 |
+
nonlocal cur_available_bytes, cur_handles
|
| 491 |
+
if not cur_handles:
|
| 492 |
+
return
|
| 493 |
+
serialized_device_handles = {device_uuid: base64.b64encode(pickle.dumps(cur_handles)).decode("utf-8")}
|
| 494 |
+
await self.update_weights_from_ipc_handles(serialized_device_handles)
|
| 495 |
+
cur_available_bytes = total_available_bytes
|
| 496 |
+
cur_handles = []
|
| 497 |
+
|
| 498 |
+
# For non-VLM, always use partial loading. For VLM, leader queries and broadcasts to all
|
| 499 |
+
# ranks in the DP replica; use get_global_rank(group, 0) since each replica has a different leader.
|
| 500 |
+
is_vlm = self.model_config.hf_config is not None and hasattr(self.model_config.hf_config, "vision_config")
|
| 501 |
+
if not is_vlm:
|
| 502 |
+
supports_partial_loading = True
|
| 503 |
+
else:
|
| 504 |
+
exclude_dp_group = self.hybrid_device_mesh["exclude_dp"].get_group()
|
| 505 |
+
spl_tensor = torch.zeros(1, dtype=torch.int32)
|
| 506 |
+
if self.is_leader_rank:
|
| 507 |
+
supports_partial_loading = await self.get_supports_partial_loading()
|
| 508 |
+
spl_tensor[0] = int(supports_partial_loading)
|
| 509 |
+
leader_global_rank = dist.get_global_rank(exclude_dp_group, 0)
|
| 510 |
+
await asyncio.to_thread(dist.broadcast, spl_tensor, src=leader_global_rank, group=exclude_dp_group)
|
| 511 |
+
supports_partial_loading = bool(spl_tensor.item())
|
| 512 |
+
|
| 513 |
+
async for name, param in ensure_async_iterator(weights):
|
| 514 |
+
if supports_partial_loading:
|
| 515 |
+
size_in_bytes = param.element_size() * param.numel()
|
| 516 |
+
if size_in_bytes > cur_available_bytes:
|
| 517 |
+
await flush()
|
| 518 |
+
|
| 519 |
+
assert cur_available_bytes >= size_in_bytes, (
|
| 520 |
+
f"cur_available_bytes: {cur_available_bytes:,} size_in_bytes: {size_in_bytes:,} name: {name}"
|
| 521 |
+
)
|
| 522 |
+
cur_available_bytes -= size_in_bytes
|
| 523 |
+
|
| 524 |
+
# Clone so the IPC handle owns the data to avoid buffer reuse
|
| 525 |
+
# before TRT-LLM reads it, which will silently corrupt weights.
|
| 526 |
+
handle = reduce_tensor(param.detach().clone())
|
| 527 |
+
cur_handles.append((name, handle))
|
| 528 |
+
|
| 529 |
+
await flush()
|
| 530 |
+
|
| 531 |
+
if self.is_leader_rank:
|
| 532 |
+
# Finalize update weights
|
| 533 |
+
await self._adapter.update_weights(None)
|
| 534 |
+
if global_steps is not None:
|
| 535 |
+
await self.server_actor.set_global_steps.remote(global_steps)
|
| 536 |
+
group = self.hybrid_device_mesh["exclude_dp"].get_group() if self.hybrid_device_mesh is not None else None
|
| 537 |
+
await asyncio.to_thread(dist.barrier, group=group)
|
| 538 |
+
|
| 539 |
+
del weights
|
| 540 |
+
gc.collect()
|
| 541 |
+
get_torch_device().empty_cache()
|
| 542 |
+
|
| 543 |
+
def _get_attribute(self, name: str):
|
| 544 |
+
return getattr(self, name)
|
repro/vendor/verl/workers/rollout/trtllm_rollout/trtllm_worker_extension.py
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 Bytedance Ltd. and/or its affiliates
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
import base64
|
| 15 |
+
import inspect
|
| 16 |
+
from typing import Optional
|
| 17 |
+
|
| 18 |
+
# Defer tensorrt_llm imports to avoid FlashInfer's check_cuda_arch() crash
|
| 19 |
+
# when this module is loaded on CPU-only Ray actors. The module is normally
|
| 20 |
+
# loaded only on GPU workers via string path in trtllm_async_server.py, but
|
| 21 |
+
# guard defensively in case of transitive imports.
|
| 22 |
+
try:
|
| 23 |
+
from tensorrt_llm import serialization
|
| 24 |
+
from tensorrt_llm._ray_utils import control_action_decorator
|
| 25 |
+
from tensorrt_llm._torch.modules.fused_moe.moe_load_balancer import MoeLoadBalancer
|
| 26 |
+
from tensorrt_llm._torch.utils import get_device_uuid
|
| 27 |
+
from tensorrt_llm.llmapi.rlhf_utils import WorkerExtension as TrtllmWorkerExtension
|
| 28 |
+
from tensorrt_llm.logger import logger
|
| 29 |
+
except (ImportError, RuntimeError):
|
| 30 |
+
# On CPU actors without CUDA, these imports may fail.
|
| 31 |
+
# The class below won't be usable, but the module can be imported safely.
|
| 32 |
+
serialization = None
|
| 33 |
+
control_action_decorator = lambda f: f # noqa: E731 — identity fallback
|
| 34 |
+
MoeLoadBalancer = None
|
| 35 |
+
get_device_uuid = None
|
| 36 |
+
TrtllmWorkerExtension = object
|
| 37 |
+
logger = None
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class WorkerExtension(TrtllmWorkerExtension):
|
| 41 |
+
def __init__(self):
|
| 42 |
+
pass
|
| 43 |
+
|
| 44 |
+
@control_action_decorator
|
| 45 |
+
def supports_partial_loading(self) -> bool:
|
| 46 |
+
"""Check if the model supports partial weight loading."""
|
| 47 |
+
try:
|
| 48 |
+
model = self.engine.model_engine.model
|
| 49 |
+
load_weights_args = inspect.getfullargspec(model.load_weights).args
|
| 50 |
+
return "allow_partial_loading" in load_weights_args
|
| 51 |
+
except Exception as e:
|
| 52 |
+
logger.warning(f"Failed to check partial loading support: {e}")
|
| 53 |
+
return False
|
| 54 |
+
|
| 55 |
+
@control_action_decorator
|
| 56 |
+
def update_weights(self, ipc_handles: Optional[dict] = None):
|
| 57 |
+
try:
|
| 58 |
+
if not hasattr(self.engine.model_engine.model, "first_pre_reload_weights"):
|
| 59 |
+
for module in self.engine.model_engine.model.modules():
|
| 60 |
+
if hasattr(module, "pre_reload_weights") and not getattr(module, "_weights_removed", False):
|
| 61 |
+
module.pre_reload_weights()
|
| 62 |
+
self.engine.model_engine.model.first_pre_reload_weights = True
|
| 63 |
+
|
| 64 |
+
if ipc_handles is not None:
|
| 65 |
+
logger.info("Update weights from IPC handles")
|
| 66 |
+
device_uuid = get_device_uuid(self.device_id)
|
| 67 |
+
|
| 68 |
+
if device_uuid not in ipc_handles:
|
| 69 |
+
raise ValueError(f"Device UUID {device_uuid} not found in ipc_handles")
|
| 70 |
+
|
| 71 |
+
weights = {}
|
| 72 |
+
|
| 73 |
+
serialized_handles = ipc_handles[device_uuid]
|
| 74 |
+
if isinstance(serialized_handles, str):
|
| 75 |
+
# Data is base64-encoded pickled bytes - deserialize it
|
| 76 |
+
# using restricted unpickler from tensorrt_llm.serialization
|
| 77 |
+
logger.info("Deserializing base64-encoded weight handles")
|
| 78 |
+
decoded_data = base64.b64decode(serialized_handles)
|
| 79 |
+
# Allow basic builtins and torch tensor reconstruction classes
|
| 80 |
+
approved_imports = {
|
| 81 |
+
"builtins": [
|
| 82 |
+
"list",
|
| 83 |
+
"tuple",
|
| 84 |
+
"str",
|
| 85 |
+
"int",
|
| 86 |
+
"float",
|
| 87 |
+
"bool",
|
| 88 |
+
"bytes",
|
| 89 |
+
"dict",
|
| 90 |
+
"NoneType",
|
| 91 |
+
"type",
|
| 92 |
+
],
|
| 93 |
+
"torch": [
|
| 94 |
+
"Tensor",
|
| 95 |
+
"FloatTensor",
|
| 96 |
+
"DoubleTensor",
|
| 97 |
+
"HalfTensor",
|
| 98 |
+
"BFloat16Tensor",
|
| 99 |
+
"IntTensor",
|
| 100 |
+
"LongTensor",
|
| 101 |
+
"ShortTensor",
|
| 102 |
+
"CharTensor",
|
| 103 |
+
"ByteTensor",
|
| 104 |
+
"BoolTensor",
|
| 105 |
+
"Size",
|
| 106 |
+
"dtype",
|
| 107 |
+
"device",
|
| 108 |
+
"float32",
|
| 109 |
+
"float16",
|
| 110 |
+
"bfloat16",
|
| 111 |
+
"int32",
|
| 112 |
+
"int64",
|
| 113 |
+
"int16",
|
| 114 |
+
"int8",
|
| 115 |
+
"uint8",
|
| 116 |
+
"bool",
|
| 117 |
+
],
|
| 118 |
+
"torch.multiprocessing.reductions": [
|
| 119 |
+
"rebuild_cuda_tensor",
|
| 120 |
+
"rebuild_tensor",
|
| 121 |
+
],
|
| 122 |
+
"torch._utils": [
|
| 123 |
+
"_rebuild_tensor_v2",
|
| 124 |
+
],
|
| 125 |
+
"torch.storage": [
|
| 126 |
+
"_load_from_bytes",
|
| 127 |
+
"_TypedStorage",
|
| 128 |
+
"UntypedStorage",
|
| 129 |
+
"TypedStorage",
|
| 130 |
+
],
|
| 131 |
+
}
|
| 132 |
+
all_handles = serialization.loads(
|
| 133 |
+
decoded_data,
|
| 134 |
+
approved_imports=approved_imports,
|
| 135 |
+
)
|
| 136 |
+
|
| 137 |
+
# Verify the result is a list as expected
|
| 138 |
+
if not isinstance(all_handles, list):
|
| 139 |
+
raise ValueError(f"Deserialized data must be a list, got {type(all_handles).__name__} instead")
|
| 140 |
+
else:
|
| 141 |
+
# Data is already in the correct format (backward compatibility)
|
| 142 |
+
all_handles = serialized_handles
|
| 143 |
+
|
| 144 |
+
for param_name, tensor_handle in all_handles:
|
| 145 |
+
func, args = tensor_handle
|
| 146 |
+
list_args = list(args)
|
| 147 |
+
list_args[6] = self.device_id
|
| 148 |
+
tensor = func(*list_args)
|
| 149 |
+
weights[param_name] = tensor
|
| 150 |
+
|
| 151 |
+
logger.info(f"weights key size: {len(weights.keys())}")
|
| 152 |
+
|
| 153 |
+
# Check if model supports partial loading and use appropriate strategy
|
| 154 |
+
model = self.engine.model_engine.model
|
| 155 |
+
load_weights_args = inspect.getfullargspec(model.load_weights).args
|
| 156 |
+
supports_partial_loading = "allow_partial_loading" in load_weights_args
|
| 157 |
+
|
| 158 |
+
if supports_partial_loading:
|
| 159 |
+
self.engine.model_engine.model_loader.reload(model, weights, allow_partial_loading=True)
|
| 160 |
+
else:
|
| 161 |
+
self.engine.model_engine.model_loader.reload(model, weights, allow_partial_loading=False)
|
| 162 |
+
else:
|
| 163 |
+
logger.info("Finalize update weights")
|
| 164 |
+
for module in self.engine.model_engine.model.modules():
|
| 165 |
+
if hasattr(module, "process_weights_after_loading") and not getattr(
|
| 166 |
+
module, "_weights_removed", False
|
| 167 |
+
):
|
| 168 |
+
module.process_weights_after_loading()
|
| 169 |
+
if hasattr(module, "post_load_weights") and not getattr(module, "_weights_removed", False):
|
| 170 |
+
module.post_load_weights()
|
| 171 |
+
moe_load_balancer = getattr(self.engine.model_engine, "moe_load_balancer", None)
|
| 172 |
+
if isinstance(moe_load_balancer, MoeLoadBalancer):
|
| 173 |
+
moe_load_balancer.register_weight_slots_after_to_cuda()
|
| 174 |
+
logger.info("moe_load_balancer finalizing model...")
|
| 175 |
+
moe_load_balancer.finalize_model()
|
| 176 |
+
logger.info("moe_load_balancer finalize model done")
|
| 177 |
+
self.engine.reset_prefix_cache()
|
| 178 |
+
delattr(self.engine.model_engine.model, "first_pre_reload_weights")
|
| 179 |
+
|
| 180 |
+
except Exception as e:
|
| 181 |
+
logger.error("Encountered an error in update_weights")
|
| 182 |
+
raise e
|
| 183 |
+
|
| 184 |
+
def reset_prefix_cache(self) -> None:
|
| 185 |
+
"""Invalidate the KV cache prefix reuse state after weight updates."""
|
| 186 |
+
self.engine.reset_prefix_cache()
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
# TODO: remove this class and revert the non-VLM path in trtllm_async_server.py
|
| 190 |
+
# to use "tensorrt_llm.llmapi.rlhf_utils.WorkerExtension" once verl's TRT-LLM version
|
| 191 |
+
# is bumped to include https://github.com/NVIDIA/TensorRT-LLM/pull/13784.
|
| 192 |
+
class RlhfWorkerExtension(TrtllmWorkerExtension):
|
| 193 |
+
"""Minimal extension of TRT-LLM's WorkerExtension for non-VLM RLHF models."""
|
| 194 |
+
|
| 195 |
+
@control_action_decorator
|
| 196 |
+
def wait_for_engine_idle(self) -> None:
|
| 197 |
+
"""Block until the engine has no active or queued requests."""
|
| 198 |
+
pass
|
repro/vendor/verl/workers/rollout/vllm_rollout/__init__.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2024 Bytedance Ltd. and/or its affiliates
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
import os
|
| 15 |
+
from importlib.metadata import PackageNotFoundError, version
|
| 16 |
+
|
| 17 |
+
from .vllm_rollout import ServerAdapter # noqa: F401
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def get_version(pkg):
|
| 21 |
+
try:
|
| 22 |
+
return version(pkg)
|
| 23 |
+
except PackageNotFoundError:
|
| 24 |
+
return None
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
vllm_package_name = "vllm"
|
| 28 |
+
vllm_package_version = get_version(vllm_package_name)
|
| 29 |
+
if vllm_package_version is None:
|
| 30 |
+
raise PackageNotFoundError(
|
| 31 |
+
"To use vllm rollout, please ensure the 'vllm' package is properly installed. See "
|
| 32 |
+
"https://verl.readthedocs.io/en/latest/start/install.html for more details"
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
if "ROCM_PATH" in os.environ:
|
| 36 |
+
import re
|
| 37 |
+
|
| 38 |
+
match = re.match(r"(\d+\.\d+\.?\d*)", vllm_package_version)
|
| 39 |
+
if match:
|
| 40 |
+
vllm_package_version = match.group(1)
|
| 41 |
+
else:
|
| 42 |
+
raise ValueError(f"Warning: Could not parse version format: {vllm_package_version}")
|
repro/vendor/verl/workers/rollout/vllm_rollout/bucketed_weight_transfer.py
ADDED
|
@@ -0,0 +1,355 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025 Bytedance Ltd. and/or its affiliates
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""
|
| 15 |
+
Bucketed weight transfer via ZMQ + IPC (or shared memory fallback).
|
| 16 |
+
|
| 17 |
+
Not recommended depending on vllm for this file.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
import logging
|
| 21 |
+
import os
|
| 22 |
+
from multiprocessing import shared_memory
|
| 23 |
+
from typing import Callable, TypedDict
|
| 24 |
+
|
| 25 |
+
import torch
|
| 26 |
+
import zmq
|
| 27 |
+
from torch.multiprocessing.reductions import reduce_tensor
|
| 28 |
+
|
| 29 |
+
from verl.utils.device import get_device_id, get_device_name, get_torch_device, is_support_ipc
|
| 30 |
+
|
| 31 |
+
logger = logging.getLogger(__file__)
|
| 32 |
+
logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "INFO"))
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class TensorMetadata(TypedDict):
|
| 36 |
+
name: str
|
| 37 |
+
shape: torch.Size
|
| 38 |
+
dtype: torch.dtype
|
| 39 |
+
offset: int
|
| 40 |
+
handle: tuple
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
# copy from https://github.com/vllm-project/vllm/blob/main/examples/offline_inference/rlhf_utils.py
|
| 44 |
+
def rebuild_ipc(handle: tuple[Callable, tuple], device_id: int | None = None) -> torch.Tensor:
|
| 45 |
+
func, args = handle
|
| 46 |
+
list_args = list(args)
|
| 47 |
+
if device_id is not None:
|
| 48 |
+
# the key is to change device id to the current device id
|
| 49 |
+
# in case two processes have different CUDA_VISIBLE_DEVICES
|
| 50 |
+
list_args[6] = device_id
|
| 51 |
+
buffer = func(*list_args)
|
| 52 |
+
return buffer
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def create_shared_memory(size: int, name: str):
|
| 56 |
+
"""Create shared memory for weight transfer. If already exists, attach to it."""
|
| 57 |
+
try:
|
| 58 |
+
shm = shared_memory.SharedMemory(name=name, create=True, size=size)
|
| 59 |
+
except FileExistsError:
|
| 60 |
+
shm = shared_memory.SharedMemory(name=name)
|
| 61 |
+
assert shm.size >= size, f"Stale shm segment '{name}': expected {size} bytes, got {shm.size}"
|
| 62 |
+
return shm
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def rebuild_shared_memory(name: str, size: int, dtype=torch.uint8):
|
| 66 |
+
"""Rebuild tensor from shared memory."""
|
| 67 |
+
shm = shared_memory.SharedMemory(name=name)
|
| 68 |
+
tensor = torch.frombuffer(shm.buf[:size], dtype=dtype)
|
| 69 |
+
|
| 70 |
+
return tensor, shm
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
class BucketedWeightSender:
|
| 74 |
+
"""
|
| 75 |
+
Send model weights via bucketed IPC transfer over ZMQ.
|
| 76 |
+
|
| 77 |
+
Packs weight tensors into a fixed-size communication buffer and sends them
|
| 78 |
+
in buckets to the receiver. Supports CUDA IPC and shared memory fallback.
|
| 79 |
+
|
| 80 |
+
Args:
|
| 81 |
+
zmq_handle: ZMQ IPC socket path (e.g., "ipc:///tmp/rl-colocate-zmq-<uuid>.sock")
|
| 82 |
+
bucket_size_mb: Communication buffer size in MB
|
| 83 |
+
use_shm: Use shared memory instead of CUDA IPC (for NPU compatibility)
|
| 84 |
+
"""
|
| 85 |
+
|
| 86 |
+
def __init__(
|
| 87 |
+
self,
|
| 88 |
+
zmq_handle: str,
|
| 89 |
+
bucket_size_mb: int = 512,
|
| 90 |
+
use_shm: bool = False,
|
| 91 |
+
):
|
| 92 |
+
self.zmq_handle = zmq_handle
|
| 93 |
+
self.bucket_size_mb = bucket_size_mb
|
| 94 |
+
self.bucket_size = int(bucket_size_mb) << 20
|
| 95 |
+
self.use_shm = use_shm
|
| 96 |
+
|
| 97 |
+
self.zmq_context = zmq.Context.instance()
|
| 98 |
+
self.socket = None
|
| 99 |
+
self.buffer = None
|
| 100 |
+
self.shm = None
|
| 101 |
+
|
| 102 |
+
async def async_send_weights(self, weights):
|
| 103 |
+
"""
|
| 104 |
+
Send weights to the receiver. Accepts a sync generator or async iterator.
|
| 105 |
+
|
| 106 |
+
Args:
|
| 107 |
+
weights: Generator or async iterator yielding (name, tensor) pairs
|
| 108 |
+
"""
|
| 109 |
+
from verl.workers.rollout.utils import ensure_async_iterator
|
| 110 |
+
|
| 111 |
+
try:
|
| 112 |
+
self._init_socket()
|
| 113 |
+
self._init_buffer()
|
| 114 |
+
|
| 115 |
+
# send bucket weights
|
| 116 |
+
offset = 0
|
| 117 |
+
bucket_meta: dict[str, TensorMetadata] = {}
|
| 118 |
+
# dtype = PrecisionType.to_dtype(self.config.dtype)
|
| 119 |
+
async for name, weight in ensure_async_iterator(weights):
|
| 120 |
+
# model parameters are in fp32 full precision
|
| 121 |
+
# (vermouth1992) we should not force cast weight here because some parameters
|
| 122 |
+
# (such as moe gate) have to keep fp32 precision. If a weight is bf16 in the rollout side,
|
| 123 |
+
# the rollout should automatically cast on demand. However, this would incur a higher weight
|
| 124 |
+
# transfer volume.
|
| 125 |
+
# weight = weight.to(dtype, non_blocking=True)
|
| 126 |
+
|
| 127 |
+
# Align each tensor before the receiver views the byte buffer
|
| 128 |
+
# using that tensor's dtype.
|
| 129 |
+
alignment = weight.element_size()
|
| 130 |
+
offset = (offset + alignment - 1) // alignment * alignment
|
| 131 |
+
|
| 132 |
+
# fill the tensor bucket
|
| 133 |
+
if offset + weight.nbytes > self.bucket_size and len(bucket_meta) > 0:
|
| 134 |
+
get_torch_device().synchronize()
|
| 135 |
+
self.socket.send_pyobj({"bucket_meta": bucket_meta, "is_last": False})
|
| 136 |
+
self.socket.recv()
|
| 137 |
+
bucket_meta = {}
|
| 138 |
+
offset = 0
|
| 139 |
+
|
| 140 |
+
if offset + weight.nbytes > self.bucket_size:
|
| 141 |
+
assert not self.use_shm, (
|
| 142 |
+
f"Weight {name}({weight.shape}, {weight.dtype}) is too large to fit in the bucket."
|
| 143 |
+
f"Please increase rollout.update_weights_bucket_megabytes({self.bucket_size_mb} MB)."
|
| 144 |
+
)
|
| 145 |
+
self._direct_send_large_weight(name, weight)
|
| 146 |
+
continue
|
| 147 |
+
|
| 148 |
+
bucket_meta[name] = {
|
| 149 |
+
"name": name,
|
| 150 |
+
"shape": weight.shape,
|
| 151 |
+
"dtype": weight.dtype,
|
| 152 |
+
"offset": offset,
|
| 153 |
+
"handle": None,
|
| 154 |
+
}
|
| 155 |
+
self.buffer[offset : offset + weight.nbytes].view(dtype=weight.dtype).view(weight.shape).copy_(
|
| 156 |
+
weight, non_blocking=True
|
| 157 |
+
)
|
| 158 |
+
offset += weight.nbytes
|
| 159 |
+
|
| 160 |
+
# send the last bucket
|
| 161 |
+
name = weight = None
|
| 162 |
+
get_torch_device().synchronize()
|
| 163 |
+
self.socket.send_pyobj({"bucket_meta": bucket_meta, "is_last": True})
|
| 164 |
+
self.socket.recv()
|
| 165 |
+
finally:
|
| 166 |
+
self._cleanup()
|
| 167 |
+
|
| 168 |
+
def _init_socket(self):
|
| 169 |
+
"""Initialize ZMQ REQ socket and bind."""
|
| 170 |
+
if self.zmq_handle.startswith("ipc://"):
|
| 171 |
+
ipc_path = self.zmq_handle[len("ipc://") :]
|
| 172 |
+
try:
|
| 173 |
+
os.remove(ipc_path)
|
| 174 |
+
except OSError:
|
| 175 |
+
pass
|
| 176 |
+
self.socket = self.zmq_context.socket(zmq.REQ)
|
| 177 |
+
self.socket.bind(self.zmq_handle)
|
| 178 |
+
|
| 179 |
+
def _init_buffer(self):
|
| 180 |
+
"""build communication buffer"""
|
| 181 |
+
buffer, shm = None, None
|
| 182 |
+
if not self.use_shm:
|
| 183 |
+
buffer = torch.empty(self.bucket_size, dtype=torch.uint8, device=f"{get_device_name()}:{get_device_id()}")
|
| 184 |
+
handle = reduce_tensor(buffer)
|
| 185 |
+
self.socket.send_pyobj(handle)
|
| 186 |
+
else:
|
| 187 |
+
import uuid
|
| 188 |
+
|
| 189 |
+
# Create unique name for shared memory
|
| 190 |
+
shm_name = f"verl_weights_{uuid.uuid4().hex}"
|
| 191 |
+
shm = create_shared_memory(self.bucket_size, shm_name)
|
| 192 |
+
buffer = torch.frombuffer(shm.buf, dtype=torch.uint8)
|
| 193 |
+
|
| 194 |
+
comm_metadata = {"name": shm_name, "size": self.bucket_size}
|
| 195 |
+
self.socket.send_pyobj(comm_metadata)
|
| 196 |
+
|
| 197 |
+
self.socket.recv()
|
| 198 |
+
self.buffer = buffer
|
| 199 |
+
self.shm = shm
|
| 200 |
+
|
| 201 |
+
def _cleanup(self):
|
| 202 |
+
"""clean up"""
|
| 203 |
+
if self.socket is not None:
|
| 204 |
+
self.socket.close()
|
| 205 |
+
self.socket = None
|
| 206 |
+
if self.zmq_handle.startswith("ipc://"):
|
| 207 |
+
ipc_path = self.zmq_handle[len("ipc://") :]
|
| 208 |
+
try:
|
| 209 |
+
os.remove(ipc_path)
|
| 210 |
+
except OSError:
|
| 211 |
+
pass
|
| 212 |
+
del self.buffer
|
| 213 |
+
self.buffer = None
|
| 214 |
+
if self.shm is not None:
|
| 215 |
+
self.shm.close()
|
| 216 |
+
self.shm.unlink()
|
| 217 |
+
del self.shm
|
| 218 |
+
self.shm = None
|
| 219 |
+
if is_support_ipc():
|
| 220 |
+
get_torch_device().ipc_collect()
|
| 221 |
+
get_torch_device().empty_cache()
|
| 222 |
+
|
| 223 |
+
def _direct_send_large_weight(self, name: str, weight: torch.Tensor):
|
| 224 |
+
"""Send a weight larger than the bucket size via cuda ipc or share memory."""
|
| 225 |
+
logger.debug(f"Direct sending large weight {name}({weight.shape}, {weight.dtype})")
|
| 226 |
+
# TODO: support fallback to shared memory
|
| 227 |
+
handle = reduce_tensor(weight)
|
| 228 |
+
bucket_meta: dict[str, TensorMetadata] = {}
|
| 229 |
+
bucket_meta[name] = {
|
| 230 |
+
"name": name,
|
| 231 |
+
"shape": weight.shape,
|
| 232 |
+
"dtype": weight.dtype,
|
| 233 |
+
"offset": 0,
|
| 234 |
+
"handle": handle,
|
| 235 |
+
}
|
| 236 |
+
self.socket.send_pyobj({"bucket_meta": bucket_meta, "is_last": False})
|
| 237 |
+
self.socket.recv()
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
class BucketedWeightReceiver:
|
| 241 |
+
"""
|
| 242 |
+
Receive model weights via bucketed IPC transfer over ZMQ.
|
| 243 |
+
|
| 244 |
+
Receives weight tensors from BucketedWeightSender and passes each
|
| 245 |
+
bucket to a callback for processing (e.g., loading into the model).
|
| 246 |
+
|
| 247 |
+
Args:
|
| 248 |
+
zmq_handle: ZMQ IPC socket path (must match sender)
|
| 249 |
+
device: Target device for received tensors
|
| 250 |
+
use_shm: Use shared memory instead of CUDA IPC
|
| 251 |
+
"""
|
| 252 |
+
|
| 253 |
+
def __init__(
|
| 254 |
+
self,
|
| 255 |
+
zmq_handle: str,
|
| 256 |
+
device: torch.device,
|
| 257 |
+
use_shm: bool = False,
|
| 258 |
+
):
|
| 259 |
+
self.zmq_handle = zmq_handle
|
| 260 |
+
self.device = device
|
| 261 |
+
self.use_shm = use_shm
|
| 262 |
+
|
| 263 |
+
self.zmq_context = zmq.Context.instance()
|
| 264 |
+
self.socket = None
|
| 265 |
+
self.buffer = None
|
| 266 |
+
self.shm = None
|
| 267 |
+
self._ack_pending = False
|
| 268 |
+
|
| 269 |
+
def receive_weights(self, on_bucket_received: callable):
|
| 270 |
+
"""
|
| 271 |
+
Receive weights from sender and process each bucket via callback.
|
| 272 |
+
|
| 273 |
+
Args:
|
| 274 |
+
on_bucket_received: Callback function(weights: list[(name, tensor)],
|
| 275 |
+
is_last: bool) called per bucket. ``is_last`` marks the final bucket
|
| 276 |
+
of a weight-sync round so consumers that need the complete tensor set
|
| 277 |
+
(e.g. vLLM ``add_lora``, which takes one adapter dict per call) can
|
| 278 |
+
defer their finalization until the whole adapter has arrived.
|
| 279 |
+
"""
|
| 280 |
+
try:
|
| 281 |
+
self._init_socket()
|
| 282 |
+
self._init_buffer()
|
| 283 |
+
|
| 284 |
+
# receive bucket and update weights
|
| 285 |
+
while True:
|
| 286 |
+
metadata = self.socket.recv_pyobj()
|
| 287 |
+
weights, tensor = [], None
|
| 288 |
+
for name, meta in metadata["bucket_meta"].items():
|
| 289 |
+
shape, dtype, offset, handle = meta["shape"], meta["dtype"], meta["offset"], meta["handle"]
|
| 290 |
+
if handle is not None:
|
| 291 |
+
tensor = rebuild_ipc(handle, self.device.index)
|
| 292 |
+
weights.append((name, tensor))
|
| 293 |
+
continue
|
| 294 |
+
size = dtype.itemsize * shape.numel()
|
| 295 |
+
tensor = self.buffer[offset : offset + size].view(dtype=dtype).view(shape)
|
| 296 |
+
if self.use_shm:
|
| 297 |
+
tensor = tensor.to(self.device)
|
| 298 |
+
weights.append((name, tensor))
|
| 299 |
+
is_last = metadata["is_last"]
|
| 300 |
+
on_bucket_received(weights, is_last)
|
| 301 |
+
get_torch_device().synchronize()
|
| 302 |
+
del weights, tensor
|
| 303 |
+
if not is_last:
|
| 304 |
+
self.socket.send(b"")
|
| 305 |
+
else:
|
| 306 |
+
self._ack_pending = True
|
| 307 |
+
break
|
| 308 |
+
finally:
|
| 309 |
+
self._cleanup()
|
| 310 |
+
|
| 311 |
+
def _init_socket(self):
|
| 312 |
+
"""Initialize ZMQ REP socket and connect."""
|
| 313 |
+
self.socket = self.zmq_context.socket(zmq.REP)
|
| 314 |
+
self.socket.connect(self.zmq_handle)
|
| 315 |
+
|
| 316 |
+
def _init_buffer(self):
|
| 317 |
+
"""Receive and rebuild communication buffer from sender."""
|
| 318 |
+
comm_metadata = self.socket.recv_pyobj()
|
| 319 |
+
buffer, shm = None, None
|
| 320 |
+
if not self.use_shm:
|
| 321 |
+
handle = comm_metadata
|
| 322 |
+
buffer = rebuild_ipc(handle, self.device.index)
|
| 323 |
+
assert buffer.dtype == torch.uint8
|
| 324 |
+
else:
|
| 325 |
+
shm_name = comm_metadata["name"]
|
| 326 |
+
shm_size = comm_metadata["size"]
|
| 327 |
+
buffer, shm = rebuild_shared_memory(shm_name, shm_size, dtype=torch.uint8)
|
| 328 |
+
self.socket.send(b"")
|
| 329 |
+
self.buffer = buffer
|
| 330 |
+
self.shm = shm
|
| 331 |
+
|
| 332 |
+
def _cleanup(self):
|
| 333 |
+
"""clean up"""
|
| 334 |
+
# Synchronize before releasing the buffer to ensure all async ops
|
| 335 |
+
# referencing it (e.g. clone, .to()) have completed.
|
| 336 |
+
get_torch_device().synchronize()
|
| 337 |
+
del self.buffer
|
| 338 |
+
self.buffer = None
|
| 339 |
+
if self.shm is not None:
|
| 340 |
+
self.shm.close()
|
| 341 |
+
del self.shm
|
| 342 |
+
self.shm = None
|
| 343 |
+
if is_support_ipc():
|
| 344 |
+
get_torch_device().ipc_collect()
|
| 345 |
+
get_torch_device().empty_cache()
|
| 346 |
+
# Ack last, after the buffer is released: the sender reclaims its bucket
|
| 347 |
+
# buffer the moment this ack returns, and CUDA IPC only frees the sender's
|
| 348 |
+
# block once our ref counter reaches zero. Acking earlier strands it in
|
| 349 |
+
# CudaIPCSentDataLimbo until some later round drains the limbo.
|
| 350 |
+
# The guard keeps the REP socket's strict recv/send alternation valid.
|
| 351 |
+
if self.socket is not None:
|
| 352 |
+
if self._ack_pending:
|
| 353 |
+
self.socket.send(b"")
|
| 354 |
+
self.socket.close()
|
| 355 |
+
self.socket = None
|
repro/vendor/verl/workers/rollout/vllm_rollout/delta_weight_transfer.py
ADDED
|
@@ -0,0 +1,347 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025 Bytedance Ltd. and/or its affiliates
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Receive ``DeltaFlush`` payloads inside vLLM workers.
|
| 15 |
+
|
| 16 |
+
VERL sends each payload through its existing ZMQ/CUDA-IPC channel. This
|
| 17 |
+
adapter decodes it and calls vLLM's checkpoint patch loader, which handles
|
| 18 |
+
checkpoint names, packed weights, and TP slicing.
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
from __future__ import annotations
|
| 22 |
+
|
| 23 |
+
import inspect
|
| 24 |
+
import json
|
| 25 |
+
from collections.abc import Iterable, Iterator
|
| 26 |
+
from dataclasses import dataclass
|
| 27 |
+
from typing import TYPE_CHECKING, Any
|
| 28 |
+
|
| 29 |
+
import torch
|
| 30 |
+
import vllm
|
| 31 |
+
|
| 32 |
+
from verl.utils.device import get_device_name, is_cuda_available
|
| 33 |
+
|
| 34 |
+
try:
|
| 35 |
+
from vllm.config.weight_transfer import WeightTransferConfig
|
| 36 |
+
from vllm.distributed.weight_transfer.base import (
|
| 37 |
+
WeightTransferEngine,
|
| 38 |
+
WeightTransferInitInfo,
|
| 39 |
+
WeightTransferUpdateInfo,
|
| 40 |
+
)
|
| 41 |
+
except ImportError as exc:
|
| 42 |
+
raise RuntimeError(
|
| 43 |
+
"vLLM delta_sharded support requires the vllm.config.weight_transfer "
|
| 44 |
+
"and vllm.distributed.weight_transfer modules"
|
| 45 |
+
) from exc
|
| 46 |
+
|
| 47 |
+
if TYPE_CHECKING:
|
| 48 |
+
from vllm.model_executor.model_loader.checkpoint_weight_patch import (
|
| 49 |
+
CheckpointWeightPatch,
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
VERL_DELTA_WEIGHT_TRANSFER_BACKEND = "verl_delta_ipc"
|
| 53 |
+
_REGISTERED = False
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _checkpoint_patch_api():
|
| 57 |
+
try:
|
| 58 |
+
from vllm.model_executor.model_loader.checkpoint_weight_patch import (
|
| 59 |
+
CheckpointWeightPatch,
|
| 60 |
+
load_checkpoint_weight_patches,
|
| 61 |
+
)
|
| 62 |
+
except ImportError as exc:
|
| 63 |
+
if exc.name != "vllm.model_executor.model_loader.checkpoint_weight_patch":
|
| 64 |
+
raise
|
| 65 |
+
raise RuntimeError(
|
| 66 |
+
"checkpoint_engine.backend='delta_sharded' with vLLM requires the checkpoint patch API from "
|
| 67 |
+
f"vLLM #50723; installed vLLM {getattr(vllm, '__version__', 'unknown')} does not provide it. "
|
| 68 |
+
"Install a compatible vLLM build or select a different checkpoint engine backend."
|
| 69 |
+
) from exc
|
| 70 |
+
return CheckpointWeightPatch, load_checkpoint_weight_patches
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def require_vllm_delta_support() -> None:
|
| 74 |
+
"""Verify that vLLM provides the required weight transfer and checkpoint patch APIs."""
|
| 75 |
+
|
| 76 |
+
init_parameters = inspect.signature(WeightTransferEngine.__init__).parameters
|
| 77 |
+
if not {"config", "vllm_config", "device", "model"}.issubset(init_parameters):
|
| 78 |
+
raise RuntimeError(
|
| 79 |
+
"checkpoint_engine.backend='delta_sharded' with vLLM requires the engine-owned WeightTransferEngine "
|
| 80 |
+
"API from vLLM #44353 and the checkpoint patch API from vLLM #50723; installed vLLM "
|
| 81 |
+
f"{getattr(vllm, '__version__', 'unknown')} does not provide the required WeightTransferEngine API. "
|
| 82 |
+
"Install a compatible vLLM build or select a different checkpoint engine backend."
|
| 83 |
+
)
|
| 84 |
+
_checkpoint_patch_api()
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def is_moe_model(hf_config) -> bool:
|
| 88 |
+
"""Whether vLLM will treat this HF config as a MoE model.
|
| 89 |
+
|
| 90 |
+
Delta updates require the Triton MoE backend, so this check must agree with
|
| 91 |
+
the engine's own MoE detection: expert counts may live under a different
|
| 92 |
+
attribute (Dbrx ``moe_num_experts``, DeepSeek ``n_routed_experts``, Mixtral
|
| 93 |
+
``num_local_experts``) or on a nested text config (multimodal MoE). The
|
| 94 |
+
fallback mirrors vLLM's attribute list for configs its convertor cannot
|
| 95 |
+
process.
|
| 96 |
+
"""
|
| 97 |
+
try:
|
| 98 |
+
from vllm.transformers_utils.config import get_hf_text_config
|
| 99 |
+
from vllm.transformers_utils.model_arch_config_convertor import (
|
| 100 |
+
MODEL_ARCH_CONFIG_CONVERTORS,
|
| 101 |
+
ModelArchConfigConvertorBase,
|
| 102 |
+
)
|
| 103 |
+
|
| 104 |
+
convertor_cls = MODEL_ARCH_CONFIG_CONVERTORS.get(
|
| 105 |
+
getattr(hf_config, "model_type", ""), ModelArchConfigConvertorBase
|
| 106 |
+
)
|
| 107 |
+
return int(convertor_cls(hf_config, get_hf_text_config(hf_config)).get_num_experts()) > 0
|
| 108 |
+
except Exception:
|
| 109 |
+
text_config = hf_config.get_text_config() if hasattr(hf_config, "get_text_config") else hf_config
|
| 110 |
+
for name in ("num_experts", "moe_num_experts", "n_routed_experts", "num_local_experts"):
|
| 111 |
+
num_experts = getattr(text_config, name, 0)
|
| 112 |
+
if isinstance(num_experts, list):
|
| 113 |
+
num_experts = num_experts[0] if num_experts else 0
|
| 114 |
+
if num_experts and int(num_experts) > 0:
|
| 115 |
+
return True
|
| 116 |
+
return False
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
@dataclass
|
| 120 |
+
class VerlDeltaIPCInitInfo(WeightTransferInitInfo):
|
| 121 |
+
"""No initialization data is needed for the local IPC transport."""
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
@dataclass
|
| 125 |
+
class VerlDeltaIPCUpdateInfo(WeightTransferUpdateInfo):
|
| 126 |
+
"""Connection details for one flush."""
|
| 127 |
+
|
| 128 |
+
zmq_handle: str | None = None
|
| 129 |
+
|
| 130 |
+
def __post_init__(self) -> None:
|
| 131 |
+
if not isinstance(self.zmq_handle, str) or not self.zmq_handle:
|
| 132 |
+
raise ValueError("VERL delta IPC updates require a worker-local zmq_handle")
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def decode_delta_payload(
|
| 136 |
+
named_tensors: Iterable[tuple[str, torch.Tensor]],
|
| 137 |
+
) -> tuple[str, list[CheckpointWeightPatch]]:
|
| 138 |
+
"""Validate and decode one DeltaFlush into vLLM checkpoint patches."""
|
| 139 |
+
|
| 140 |
+
from verl.checkpoint_engine.delta_sync.encode import checksum
|
| 141 |
+
|
| 142 |
+
CheckpointWeightPatch, _ = _checkpoint_patch_api()
|
| 143 |
+
tensors = dict(named_tensors)
|
| 144 |
+
try:
|
| 145 |
+
spec_tensor = tensors["__delta_spec__"]
|
| 146 |
+
values = tensors["__values__"]
|
| 147 |
+
spec = json.loads(bytes(spec_tensor.detach().cpu().numpy().tobytes()).decode())
|
| 148 |
+
except KeyError as exc:
|
| 149 |
+
raise ValueError(f"DeltaFlush is missing {exc.args[0]}") from exc
|
| 150 |
+
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
| 151 |
+
raise ValueError("DeltaFlush contains an invalid JSON spec") from exc
|
| 152 |
+
|
| 153 |
+
encoding = spec["encoding"]
|
| 154 |
+
if encoding not in {"dense", "indices"}:
|
| 155 |
+
raise ValueError(f"Unsupported DeltaFlush encoding: {encoding!r}")
|
| 156 |
+
positions = tensors.get("__positions__")
|
| 157 |
+
if positions is None:
|
| 158 |
+
positions = torch.empty(0, dtype=torch.uint8, device=values.device)
|
| 159 |
+
if encoding == "dense" and positions.numel():
|
| 160 |
+
raise ValueError("Dense DeltaFlush must not contain positions")
|
| 161 |
+
if encoding == "indices" and "__positions__" not in tensors:
|
| 162 |
+
raise ValueError("Indices DeltaFlush requires __positions__")
|
| 163 |
+
|
| 164 |
+
got_checksum = checksum(positions, values)
|
| 165 |
+
expected_checksum = int(spec["checksum"])
|
| 166 |
+
if got_checksum != expected_checksum:
|
| 167 |
+
raise RuntimeError(
|
| 168 |
+
f"Delta checksum mismatch in vLLM consumer: got {got_checksum}, expected {expected_checksum}"
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
+
patches: list[CheckpointWeightPatch] = []
|
| 172 |
+
for param_spec in spec["params"]:
|
| 173 |
+
name = param_spec["name"]
|
| 174 |
+
dtype = getattr(torch, param_spec["dtype"])
|
| 175 |
+
patch_values = values[param_spec["val_start"] : param_spec["val_end"]]
|
| 176 |
+
if encoding == "dense":
|
| 177 |
+
patch_indices = None
|
| 178 |
+
else:
|
| 179 |
+
if param_spec["pos_width"] != 4:
|
| 180 |
+
raise ValueError(f"{name}: indices encoding requires pos_width=4")
|
| 181 |
+
pos_bytes = positions[param_spec["pos_start"] : param_spec["pos_end"]]
|
| 182 |
+
if pos_bytes.numel() != patch_values.numel() * 4:
|
| 183 |
+
raise ValueError(f"{name}: position and value slices have inconsistent lengths")
|
| 184 |
+
patch_indices = pos_bytes.view(torch.int32)
|
| 185 |
+
|
| 186 |
+
patches.append(
|
| 187 |
+
CheckpointWeightPatch(
|
| 188 |
+
name=name,
|
| 189 |
+
shape=tuple(param_spec["shape"]),
|
| 190 |
+
dtype=dtype,
|
| 191 |
+
values=patch_values,
|
| 192 |
+
indices=patch_indices,
|
| 193 |
+
)
|
| 194 |
+
)
|
| 195 |
+
return encoding, patches
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
class VerlDeltaIPCWeightTransferEngine(WeightTransferEngine):
|
| 199 |
+
"""Consume VERL DeltaFlush payloads over the colocated CUDA-IPC channel."""
|
| 200 |
+
|
| 201 |
+
init_info_cls = VerlDeltaIPCInitInfo
|
| 202 |
+
update_info_cls = VerlDeltaIPCUpdateInfo
|
| 203 |
+
supports_draft_weight_update = False
|
| 204 |
+
|
| 205 |
+
def __init__(
|
| 206 |
+
self,
|
| 207 |
+
config: WeightTransferConfig,
|
| 208 |
+
vllm_config,
|
| 209 |
+
device: torch.device,
|
| 210 |
+
model: torch.nn.Module,
|
| 211 |
+
) -> None:
|
| 212 |
+
super().__init__(config, vllm_config, device, model)
|
| 213 |
+
self._session_encoding: str | None = None
|
| 214 |
+
self._update_failed = False
|
| 215 |
+
|
| 216 |
+
def _validate_configuration(self) -> None:
|
| 217 |
+
require_vllm_delta_support()
|
| 218 |
+
if not is_cuda_available or self.device.type != get_device_name():
|
| 219 |
+
raise NotImplementedError("VERL delta weight transfer requires CUDA")
|
| 220 |
+
if self.parallel_config.data_parallel_size != 1:
|
| 221 |
+
raise NotImplementedError("VERL delta weight transfer requires data_parallel_size=1")
|
| 222 |
+
if self.parallel_config.pipeline_parallel_size != 1:
|
| 223 |
+
raise NotImplementedError("VERL delta weight transfer requires pipeline_parallel_size=1")
|
| 224 |
+
if self.model_config.dtype != torch.bfloat16:
|
| 225 |
+
raise NotImplementedError("VERL delta weight transfer supports only BF16 vLLM models")
|
| 226 |
+
if getattr(self.vllm_config, "quant_config", None) is not None:
|
| 227 |
+
raise NotImplementedError("VERL delta weight transfer does not support quantized vLLM models")
|
| 228 |
+
if getattr(self.vllm_config, "speculative_config", None) is not None:
|
| 229 |
+
raise NotImplementedError("VERL delta weight transfer does not support speculative decoding")
|
| 230 |
+
|
| 231 |
+
def init_transfer_engine(self, init_info: VerlDeltaIPCInitInfo) -> None:
|
| 232 |
+
self._validate_configuration()
|
| 233 |
+
|
| 234 |
+
def start_weight_update(self) -> None:
|
| 235 |
+
if self._update_failed:
|
| 236 |
+
raise RuntimeError(
|
| 237 |
+
"A previous delta update did not complete; restart the full job so every rollout worker receives a "
|
| 238 |
+
"fresh dense seed. Restarting only the vLLM workers is unsafe because the producer snapshot may "
|
| 239 |
+
"already have advanced."
|
| 240 |
+
)
|
| 241 |
+
self._session_encoding = None
|
| 242 |
+
|
| 243 |
+
def _receive_payload(self, *, zmq_handle: str) -> list[tuple[str, torch.Tensor]]:
|
| 244 |
+
from verl.workers.rollout.vllm_rollout.bucketed_weight_transfer import (
|
| 245 |
+
BucketedWeightReceiver,
|
| 246 |
+
)
|
| 247 |
+
|
| 248 |
+
receiver = BucketedWeightReceiver(
|
| 249 |
+
zmq_handle=zmq_handle,
|
| 250 |
+
device=self.device,
|
| 251 |
+
use_shm=False,
|
| 252 |
+
)
|
| 253 |
+
payload: list[tuple[str, torch.Tensor]] = []
|
| 254 |
+
callback_error: BaseException | None = None
|
| 255 |
+
|
| 256 |
+
def retain_bucket(weights: list[tuple[str, torch.Tensor]], _is_last: bool) -> None:
|
| 257 |
+
nonlocal callback_error
|
| 258 |
+
if callback_error is not None:
|
| 259 |
+
return
|
| 260 |
+
try:
|
| 261 |
+
# The receiver reuses its IPC buffer for the next bucket, so
|
| 262 |
+
# retain a private copy until this flush is decoded and applied.
|
| 263 |
+
payload.extend((name, tensor.clone()) for name, tensor in weights)
|
| 264 |
+
except BaseException as exc:
|
| 265 |
+
# Finish the receiver handshake before reporting a callback error.
|
| 266 |
+
callback_error = exc
|
| 267 |
+
|
| 268 |
+
receiver.receive_weights(on_bucket_received=retain_bucket)
|
| 269 |
+
if callback_error is not None:
|
| 270 |
+
raise callback_error
|
| 271 |
+
return payload
|
| 272 |
+
|
| 273 |
+
def receive_weights(self, update_info: VerlDeltaIPCUpdateInfo) -> None:
|
| 274 |
+
assert update_info.zmq_handle is not None
|
| 275 |
+
try:
|
| 276 |
+
payload = self._receive_payload(zmq_handle=update_info.zmq_handle)
|
| 277 |
+
encoding, patches = decode_delta_payload(payload)
|
| 278 |
+
|
| 279 |
+
if self._session_encoding is None:
|
| 280 |
+
self._session_encoding = encoding
|
| 281 |
+
if encoding == "dense":
|
| 282 |
+
from vllm.model_executor.model_loader.reload import (
|
| 283 |
+
initialize_layerwise_reload,
|
| 284 |
+
)
|
| 285 |
+
|
| 286 |
+
initialize_layerwise_reload(self.model)
|
| 287 |
+
elif encoding != self._session_encoding:
|
| 288 |
+
raise ValueError(
|
| 289 |
+
"One weight update cannot mix dense and sparse DeltaFlush payloads "
|
| 290 |
+
f"({self._session_encoding!r} then {encoding!r})"
|
| 291 |
+
)
|
| 292 |
+
|
| 293 |
+
_, load_checkpoint_weight_patches = _checkpoint_patch_api()
|
| 294 |
+
# The trusted producer emits every changed position at most once;
|
| 295 |
+
# skip vLLM's sort-based duplicate check on the rollout GPU.
|
| 296 |
+
load_checkpoint_weight_patches(
|
| 297 |
+
self.model,
|
| 298 |
+
patches,
|
| 299 |
+
validate_unique_indices=False,
|
| 300 |
+
)
|
| 301 |
+
except BaseException:
|
| 302 |
+
# An earlier flush may already have changed runtime weights, so a
|
| 303 |
+
# later update would start from an unknown partial state.
|
| 304 |
+
self._update_failed = True
|
| 305 |
+
raise
|
| 306 |
+
|
| 307 |
+
def finish_weight_update(self) -> None:
|
| 308 |
+
if self._session_encoding == "dense":
|
| 309 |
+
# Dense updates use vLLM's layerwise reload lifecycle. Finalize it
|
| 310 |
+
# once after the last flush to run model-specific post-load work.
|
| 311 |
+
from vllm.model_executor.model_loader.reload import (
|
| 312 |
+
finalize_layerwise_reload,
|
| 313 |
+
)
|
| 314 |
+
|
| 315 |
+
try:
|
| 316 |
+
finalize_layerwise_reload(self.model, self.model_config)
|
| 317 |
+
except BaseException:
|
| 318 |
+
self._update_failed = True
|
| 319 |
+
raise
|
| 320 |
+
# Sparse updates write to initialized runtime tensors, so there is
|
| 321 |
+
# nothing to rebuild after the last flush.
|
| 322 |
+
self._session_encoding = None
|
| 323 |
+
|
| 324 |
+
def shutdown(self) -> None:
|
| 325 |
+
self._session_encoding = None
|
| 326 |
+
|
| 327 |
+
@staticmethod
|
| 328 |
+
def trainer_send_weights(
|
| 329 |
+
iterator: Iterator[Any],
|
| 330 |
+
trainer_args: dict[str, Any] | Any,
|
| 331 |
+
) -> None:
|
| 332 |
+
raise NotImplementedError("VERL sends DeltaFlush payloads; this adapter only receives them")
|
| 333 |
+
|
| 334 |
+
|
| 335 |
+
def register_verl_delta_weight_transfer_engine() -> None:
|
| 336 |
+
"""Register the adapter before vLLM creates its worker-side engine."""
|
| 337 |
+
|
| 338 |
+
global _REGISTERED
|
| 339 |
+
if _REGISTERED:
|
| 340 |
+
return
|
| 341 |
+
from vllm.distributed.weight_transfer import WeightTransferEngineFactory
|
| 342 |
+
|
| 343 |
+
WeightTransferEngineFactory.register_engine(
|
| 344 |
+
VERL_DELTA_WEIGHT_TRANSFER_BACKEND,
|
| 345 |
+
VerlDeltaIPCWeightTransferEngine,
|
| 346 |
+
)
|
| 347 |
+
_REGISTERED = True
|
repro/vendor/verl/workers/rollout/vllm_rollout/utils.py
ADDED
|
@@ -0,0 +1,573 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2024 Bytedance Ltd. and/or its affiliates
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
import ctypes
|
| 15 |
+
import dataclasses
|
| 16 |
+
import functools
|
| 17 |
+
import json
|
| 18 |
+
import logging
|
| 19 |
+
import os
|
| 20 |
+
import platform
|
| 21 |
+
import signal
|
| 22 |
+
import threading
|
| 23 |
+
from collections.abc import Mapping
|
| 24 |
+
from types import MethodType
|
| 25 |
+
from typing import Any, Literal, Optional, get_args
|
| 26 |
+
|
| 27 |
+
import torch
|
| 28 |
+
from vllm.outputs import RequestOutput
|
| 29 |
+
|
| 30 |
+
from verl.utils.device import get_device_name, is_npu_available
|
| 31 |
+
from verl.utils.vllm import TensorLoRARequest, VLLMHijack, resolve_weight_name
|
| 32 |
+
from verl.utils.vllm.patch import patch_vllm_moe_model_weight_loader
|
| 33 |
+
from verl.utils.vllm.vllm_quant_utils import apply_vllm_quant_patches, is_quantized_model, load_quanted_weights
|
| 34 |
+
from verl.workers.rollout.vllm_rollout.weight_update_utils import apply_buffer_updates, split_buffer_updates
|
| 35 |
+
|
| 36 |
+
logger = logging.getLogger(__file__)
|
| 37 |
+
logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
|
| 38 |
+
|
| 39 |
+
# magic numbers that ensure we are using the same LoRA adapter during the rollout and training process
|
| 40 |
+
VLLM_LORA_INT_ID = 123
|
| 41 |
+
VLLM_LORA_NAME = "123"
|
| 42 |
+
VLLM_LORA_PATH = "simon_lora_path"
|
| 43 |
+
|
| 44 |
+
VLLM_ASCEND_REQUIRED_ENV_VARS = {"VLLM_ALL2ALL_BACKEND": "flashinfer_all2allv", "VLLM_ASCEND_ENABLE_NZ": "0"}
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _resolve_vllm_weight_sync_local_rank(worker_local_rank: int, parallel_config: Any) -> int:
|
| 48 |
+
worker_local_rank = int(worker_local_rank)
|
| 49 |
+
if parallel_config is None:
|
| 50 |
+
return worker_local_rank
|
| 51 |
+
|
| 52 |
+
tp_size = max(int(getattr(parallel_config, "tensor_parallel_size", 1) or 1), 1)
|
| 53 |
+
dp_size = int(getattr(parallel_config, "data_parallel_size", 1) or 1)
|
| 54 |
+
dp_local_size = int(getattr(parallel_config, "data_parallel_size_local", 1) or 1)
|
| 55 |
+
if dp_size <= 1 and dp_local_size <= 1:
|
| 56 |
+
return worker_local_rank
|
| 57 |
+
|
| 58 |
+
dp_local_rank = getattr(parallel_config, "data_parallel_rank_local", None)
|
| 59 |
+
if dp_local_rank is None:
|
| 60 |
+
dp_rank = getattr(parallel_config, "data_parallel_rank", None)
|
| 61 |
+
if dp_rank is None:
|
| 62 |
+
dp_rank = getattr(parallel_config, "data_parallel_index", None)
|
| 63 |
+
if dp_rank is not None and dp_local_size > 0:
|
| 64 |
+
dp_local_rank = int(dp_rank) % dp_local_size
|
| 65 |
+
|
| 66 |
+
if dp_local_rank is None:
|
| 67 |
+
return worker_local_rank
|
| 68 |
+
|
| 69 |
+
tp_rank = worker_local_rank % tp_size
|
| 70 |
+
return int(dp_local_rank) * tp_size + tp_rank
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def set_death_signal():
|
| 74 |
+
"""Kill the current process when the parent process exits."""
|
| 75 |
+
if platform.system() != "Linux":
|
| 76 |
+
return
|
| 77 |
+
libc = ctypes.CDLL("libc.so.6")
|
| 78 |
+
libc.prctl(1, signal.SIGKILL)
|
| 79 |
+
if os.getppid() == 1:
|
| 80 |
+
os.kill(os.getpid(), signal.SIGKILL)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def get_vllm_max_lora_rank(lora_rank: int):
|
| 84 |
+
"""
|
| 85 |
+
For vLLM, automatically adjusts the `max_lora_rank` to the nearest allowed value.
|
| 86 |
+
The allowed values are retrieved from vLLM's MaxLoRARanks type definition.
|
| 87 |
+
"""
|
| 88 |
+
assert lora_rank > 0, f"lora_rank must be greater than 0, get {lora_rank}"
|
| 89 |
+
|
| 90 |
+
try:
|
| 91 |
+
from vllm.config.lora import MaxLoRARanks
|
| 92 |
+
except Exception:
|
| 93 |
+
# FIXME: migrate vllm version https://github.com/vllm-project/vllm/blob/main/vllm/config/lora.py#L25
|
| 94 |
+
MaxLoRARanks = Literal[1, 8, 16, 32, 64, 128, 256, 320, 512]
|
| 95 |
+
|
| 96 |
+
vllm_max_lora_ranks = sorted(get_args(MaxLoRARanks))
|
| 97 |
+
if lora_rank > vllm_max_lora_ranks[-1]:
|
| 98 |
+
raise ValueError(f"lora_rank must be less than or equal to {vllm_max_lora_ranks[-1]}, but got {lora_rank}")
|
| 99 |
+
|
| 100 |
+
for rank in vllm_max_lora_ranks:
|
| 101 |
+
if lora_rank <= rank:
|
| 102 |
+
return rank
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
# https://github.com/vllm-project/vllm/issues/13175
|
| 106 |
+
def monkey_patch_compute_logits(model, vocab_size: int, banned_token_ids: Optional[list[int]] = None):
|
| 107 |
+
"""Mask the tokens the sampler must never pick.
|
| 108 |
+
|
| 109 |
+
Beyond the out-of-vocabulary tail, `banned_token_ids` covers tokens that live *inside* the
|
| 110 |
+
vocabulary yet are still illegal to generate: the vision placeholders, which are meaningless
|
| 111 |
+
unless a real image or video sits behind them. See `get_vision_placeholder_token_ids`.
|
| 112 |
+
"""
|
| 113 |
+
original_compute_logits = model.compute_logits
|
| 114 |
+
# Built once and cached on the device: compute_logits runs on every decode step, and
|
| 115 |
+
# rebuilding the index there costs a host-to-device copy per step.
|
| 116 |
+
banned_index = torch.tensor(banned_token_ids, dtype=torch.long) if banned_token_ids else None
|
| 117 |
+
|
| 118 |
+
def compute_logits(
|
| 119 |
+
self,
|
| 120 |
+
*args,
|
| 121 |
+
**kwargs,
|
| 122 |
+
) -> torch.Tensor:
|
| 123 |
+
nonlocal banned_index
|
| 124 |
+
|
| 125 |
+
logits = original_compute_logits(*args, **kwargs)
|
| 126 |
+
logits[..., vocab_size:] = float("-inf")
|
| 127 |
+
if banned_index is not None:
|
| 128 |
+
if banned_index.device != logits.device:
|
| 129 |
+
banned_index = banned_index.to(logits.device)
|
| 130 |
+
logits.index_fill_(-1, banned_index, float("-inf"))
|
| 131 |
+
return logits
|
| 132 |
+
|
| 133 |
+
model.compute_logits = MethodType(compute_logits, model)
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
class vLLMColocateWorkerExtension:
|
| 137 |
+
"""
|
| 138 |
+
The class for vLLM's worker to inherit from, in the colocate setting.
|
| 139 |
+
By defining an extension class, the code can work no matter what is
|
| 140 |
+
the underlying worker class. This way, the code can be compatible
|
| 141 |
+
with both vLLM V0 and V1.
|
| 142 |
+
NOTE: we define this class in a separate module, and the main module
|
| 143 |
+
should pass the full qualified name as `worker_extension_cls` argument.
|
| 144 |
+
|
| 145 |
+
Feature support:
|
| 146 |
+
1. LoRA
|
| 147 |
+
2. Online FP8 quantization
|
| 148 |
+
"""
|
| 149 |
+
|
| 150 |
+
def __new__(cls, **kwargs):
|
| 151 |
+
set_death_signal()
|
| 152 |
+
|
| 153 |
+
if os.environ.get("VERL_FULL_DETERMINISM", "0") == "1":
|
| 154 |
+
from verl.workers.engine.utils import enable_full_determinism
|
| 155 |
+
|
| 156 |
+
# VERL_SEED is set by vLLMHttpServer.__init__ only when the
|
| 157 |
+
# rollout config has full_determinism=true. Worker sub-processes
|
| 158 |
+
# inherit their parent's env, so rollout workers will see it but
|
| 159 |
+
# RM workers (whose parent vLLMHttpServer does not set it) won't.
|
| 160 |
+
# If VERL_SEED is missing, skip — RM doesn't need the determinism
|
| 161 |
+
# patch, only rollout does.
|
| 162 |
+
verl_seed = os.environ.get("VERL_SEED")
|
| 163 |
+
if verl_seed is not None:
|
| 164 |
+
enable_full_determinism(seed=int(verl_seed))
|
| 165 |
+
|
| 166 |
+
# 1. patch for Lora
|
| 167 |
+
VLLMHijack.hijack()
|
| 168 |
+
vllm_config = kwargs.get("vllm_config")
|
| 169 |
+
weight_transfer_config = getattr(vllm_config, "weight_transfer_config", None)
|
| 170 |
+
if getattr(weight_transfer_config, "backend", None) == "verl_delta_ipc":
|
| 171 |
+
from verl.workers.rollout.vllm_rollout.delta_weight_transfer import (
|
| 172 |
+
register_verl_delta_weight_transfer_engine,
|
| 173 |
+
)
|
| 174 |
+
|
| 175 |
+
register_verl_delta_weight_transfer_engine()
|
| 176 |
+
# 2. patch online fp8 quant. Some models, including DeepSeek-V4, get
|
| 177 |
+
# fp8 from the HF config rather than an explicit rollout quantization arg.
|
| 178 |
+
if os.environ.get("VERL_VLLM_FP8_QUANT_ENABLED", "0") == "1" or is_quantized_model(vllm_config):
|
| 179 |
+
apply_vllm_quant_patches()
|
| 180 |
+
# 3. patch QAT (compressed-tensors NVFP4) for dynamic weight loading
|
| 181 |
+
quant_config = getattr(vllm_config, "quant_config", None) if vllm_config else None
|
| 182 |
+
_is_qat_model = getattr(quant_config, "quant_format", None) == "nvfp4-pack-quantized"
|
| 183 |
+
_is_modelopt_qat = type(quant_config).__name__ == "ModelOptNvFp4Config"
|
| 184 |
+
if _is_qat_model:
|
| 185 |
+
from verl.utils.qat import apply_qat_patches
|
| 186 |
+
|
| 187 |
+
apply_qat_patches()
|
| 188 |
+
logger.info("Applied QAT (compressed-tensors) patches in vLLM worker subprocess")
|
| 189 |
+
elif _is_modelopt_qat:
|
| 190 |
+
from verl.utils.modelopt import apply_modelopt_nvfp4_patches
|
| 191 |
+
|
| 192 |
+
apply_modelopt_nvfp4_patches()
|
| 193 |
+
logger.info("Applied ModelOpt NVFP4 patches in vLLM worker subprocess")
|
| 194 |
+
|
| 195 |
+
# TODO: For ascend NPU, when the corresponding vllm-ascend version is upgraded to v0.13.0,
|
| 196 |
+
# please remove the VLLM_ASCEND_REQUIRED_ENV_VARS variable replacement action.
|
| 197 |
+
# This is only a fix for vllm version < v0.13.0.
|
| 198 |
+
if is_npu_available:
|
| 199 |
+
for k in VLLM_ASCEND_REQUIRED_ENV_VARS:
|
| 200 |
+
if k not in os.environ:
|
| 201 |
+
os.environ[k] = VLLM_ASCEND_REQUIRED_ENV_VARS[k]
|
| 202 |
+
|
| 203 |
+
instance = super().__new__(cls)
|
| 204 |
+
instance._is_qat_model = _is_qat_model
|
| 205 |
+
instance._is_modelopt_qat = _is_modelopt_qat
|
| 206 |
+
return instance
|
| 207 |
+
|
| 208 |
+
def _get_drafter_model(self):
|
| 209 |
+
"""Return the drafter's model object, or None if unavailable."""
|
| 210 |
+
drafter = getattr(self.model_runner, "drafter", None)
|
| 211 |
+
return drafter.model if drafter is not None and hasattr(drafter, "model") else None
|
| 212 |
+
|
| 213 |
+
def _get_draft_model_config(self):
|
| 214 |
+
"""Return the draft model config from speculative_config, or None."""
|
| 215 |
+
spec = self.model_runner.vllm_config.speculative_config
|
| 216 |
+
return spec.draft_model_config if spec is not None and spec.draft_model_config is not None else None
|
| 217 |
+
|
| 218 |
+
def _use_mtp_drafter_weight_sync(self):
|
| 219 |
+
"""Return whether the vLLM MTP drafter should receive actor weights."""
|
| 220 |
+
spec = self.model_runner.vllm_config.speculative_config
|
| 221 |
+
return spec is not None and spec.method == "mtp" and self._get_drafter_model() is not None
|
| 222 |
+
|
| 223 |
+
def _iter_all_models(self):
|
| 224 |
+
"""Yield models that need weight updates.
|
| 225 |
+
|
| 226 |
+
Only vLLM MTP drafter sync is supported for now. Independent non-MTP
|
| 227 |
+
draft models are not compatible with actor weight loading through this path.
|
| 228 |
+
"""
|
| 229 |
+
yield self.model_runner.model
|
| 230 |
+
if self._use_mtp_drafter_weight_sync():
|
| 231 |
+
yield self._get_drafter_model()
|
| 232 |
+
|
| 233 |
+
def _iter_all_models_with_config(self):
|
| 234 |
+
"""Yield (model, model_config) for models that need post-processing."""
|
| 235 |
+
yield self.model_runner.model, self.model_runner.vllm_config.model_config
|
| 236 |
+
if self._use_mtp_drafter_weight_sync():
|
| 237 |
+
draft_cfg = self._get_draft_model_config()
|
| 238 |
+
if draft_cfg is not None:
|
| 239 |
+
yield self._get_drafter_model(), draft_cfg
|
| 240 |
+
|
| 241 |
+
def monkey_patch_model(self, vocab_size: int, banned_token_ids: Optional[list[int]] = None):
|
| 242 |
+
for model in self._iter_all_models():
|
| 243 |
+
# patch compute_logits to avoid sampling OOV and other illegal tokens
|
| 244 |
+
monkey_patch_compute_logits(model, vocab_size, banned_token_ids)
|
| 245 |
+
# patch weight loader to support MoE model
|
| 246 |
+
patch_vllm_moe_model_weight_loader(model)
|
| 247 |
+
|
| 248 |
+
def update_weights_from_ipc(self, peft_config: dict = None, base_sync_done=False, use_shm: bool = False):
|
| 249 |
+
"""Update the weights of the rollout model."""
|
| 250 |
+
from verl.workers.rollout.vllm_rollout.bucketed_weight_transfer import BucketedWeightReceiver
|
| 251 |
+
|
| 252 |
+
if self.device is None:
|
| 253 |
+
# vLLM workers may leave self.device unset on non-CUDA platforms (e.g. NPU);
|
| 254 |
+
# fall back to the worker's local rank on the current accelerator.
|
| 255 |
+
self.device = torch.device(f"{get_device_name()}:{self.local_rank}")
|
| 256 |
+
|
| 257 |
+
# =========================== step 1: prepare for weight loading ===========================
|
| 258 |
+
quant_reload_states = None
|
| 259 |
+
|
| 260 |
+
# The engine came up on dummy weights, whose init zeroes integer buffers on
|
| 261 |
+
# ROCm -- including the expert-parallel routing maps, which no weight stream
|
| 262 |
+
# restores. Repair them before the reload so the rollout routes correctly.
|
| 263 |
+
if torch.version.hip is not None:
|
| 264 |
+
from verl.utils.vllm.rocm_vllm_moe_expert_map import restore_moe_expert_maps
|
| 265 |
+
|
| 266 |
+
for model in self._iter_all_models():
|
| 267 |
+
restore_moe_expert_maps(model)
|
| 268 |
+
|
| 269 |
+
if self._is_qat_model:
|
| 270 |
+
# QAT (compressed-tensors): Prepare for weight loading BEFORE receiving any buckets
|
| 271 |
+
from verl.utils.qat import prepare_qat_for_load_weights
|
| 272 |
+
|
| 273 |
+
for model in self._iter_all_models():
|
| 274 |
+
prepare_qat_for_load_weights(model, device=self.device)
|
| 275 |
+
logger.info("QAT: prepare_qat_for_load_weights completed")
|
| 276 |
+
elif self._is_modelopt_qat:
|
| 277 |
+
from verl.utils.modelopt.vllm_modelopt_patch import prepare_modelopt_for_weight_reload
|
| 278 |
+
|
| 279 |
+
prepare_modelopt_for_weight_reload(self.model_runner.model, device=self.device)
|
| 280 |
+
logger.info("ModelOpt: prepare_modelopt_for_weight_reload completed")
|
| 281 |
+
elif peft_config and base_sync_done:
|
| 282 |
+
# Remove the old LoRA before the new one arrives (applied after is_last below).
|
| 283 |
+
self.remove_lora(VLLM_LORA_INT_ID)
|
| 284 |
+
logger.info("LoRA adapter sync: remove old lora and prepare new lora")
|
| 285 |
+
elif is_quantized_model(self.model_runner.vllm_config):
|
| 286 |
+
from verl.utils.vllm.vllm_quant_utils import prepare_quanted_weights_for_loading
|
| 287 |
+
|
| 288 |
+
quant_reload_states = [
|
| 289 |
+
(model, prepare_quanted_weights_for_loading(model)) for model in self._iter_all_models()
|
| 290 |
+
]
|
| 291 |
+
else:
|
| 292 |
+
# TODO(wuxibin): not need anymore for newer vllm version.
|
| 293 |
+
for model in self._iter_all_models():
|
| 294 |
+
patch_vllm_moe_model_weight_loader(model)
|
| 295 |
+
|
| 296 |
+
# =========================== step 2: receive weights and update ===========================
|
| 297 |
+
receiver = BucketedWeightReceiver(
|
| 298 |
+
zmq_handle=self._get_zmq_handle(),
|
| 299 |
+
device=self.device,
|
| 300 |
+
use_shm=use_shm,
|
| 301 |
+
)
|
| 302 |
+
# LoRA adapters need a single complete tensor dict per ``add_lora``, but
|
| 303 |
+
# the bucketed transport may split one across buckets. Accumulate and
|
| 304 |
+
# apply only after ``is_last``; standard base weights load per bucket.
|
| 305 |
+
lora_weights: dict[str, torch.Tensor] | None = {} if (peft_config and base_sync_done) else None
|
| 306 |
+
|
| 307 |
+
def on_bucket_received(weights: list[tuple[str, torch.Tensor]], is_last: bool) -> None:
|
| 308 |
+
if lora_weights is not None:
|
| 309 |
+
# Clone: add_lora keeps these past the callback (reused IPC buffer, #6454).
|
| 310 |
+
lora_weights.update((name, tensor.clone()) for name, tensor in weights)
|
| 311 |
+
if not is_last:
|
| 312 |
+
return
|
| 313 |
+
self._update_weights(
|
| 314 |
+
list(lora_weights.items()),
|
| 315 |
+
peft_config=peft_config,
|
| 316 |
+
base_sync_done=base_sync_done,
|
| 317 |
+
)
|
| 318 |
+
lora_weights.clear()
|
| 319 |
+
return
|
| 320 |
+
self._update_weights(
|
| 321 |
+
weights,
|
| 322 |
+
peft_config=peft_config,
|
| 323 |
+
base_sync_done=base_sync_done,
|
| 324 |
+
)
|
| 325 |
+
|
| 326 |
+
receiver.receive_weights(on_bucket_received=on_bucket_received)
|
| 327 |
+
|
| 328 |
+
# =========================== step 3: process weights after loading ===========================
|
| 329 |
+
if self._is_qat_model:
|
| 330 |
+
# QAT (compressed-tensors): call process_weights_after_loading AFTER all buckets are received
|
| 331 |
+
from verl.utils.qat import manual_process_weights_after_loading
|
| 332 |
+
|
| 333 |
+
for model in self._iter_all_models():
|
| 334 |
+
manual_process_weights_after_loading(model)
|
| 335 |
+
logger.info("QAT: process_weights_after_loading completed")
|
| 336 |
+
elif self._is_modelopt_qat:
|
| 337 |
+
from verl.utils.modelopt.vllm_modelopt_patch import modelopt_process_weights_after_loading
|
| 338 |
+
|
| 339 |
+
modelopt_process_weights_after_loading(self.model_runner.model)
|
| 340 |
+
logger.info("ModelOpt QAT: process_weights_after_loading completed")
|
| 341 |
+
elif peft_config and base_sync_done:
|
| 342 |
+
logger.info("LoRA adapter sync, no post-process needed")
|
| 343 |
+
elif is_quantized_model(self.model_runner.vllm_config):
|
| 344 |
+
from verl.utils.vllm.vllm_quant_utils import process_quanted_weights_after_loading
|
| 345 |
+
|
| 346 |
+
for model, reload_state in quant_reload_states:
|
| 347 |
+
process_quanted_weights_after_loading(model, reload_state)
|
| 348 |
+
else:
|
| 349 |
+
# Some post-load transforms are non-idempotent; run once after all buckets.
|
| 350 |
+
from vllm.model_executor.model_loader.utils import process_weights_after_loading
|
| 351 |
+
|
| 352 |
+
for model, model_config in self._iter_all_models_with_config():
|
| 353 |
+
process_weights_after_loading(model, model_config, self.device)
|
| 354 |
+
|
| 355 |
+
def _apply_buffer_updates_all_models(self, buffer_updates, main_named_buffers):
|
| 356 |
+
"""Apply buffer updates to the main model and any synced MTP drafter.
|
| 357 |
+
|
| 358 |
+
The main model (yielded first) reuses the prebuilt ``named_buffers`` map;
|
| 359 |
+
the drafter builds its own. Returns buffers applied to the main model.
|
| 360 |
+
"""
|
| 361 |
+
models = list(self._iter_all_models())
|
| 362 |
+
loaded = apply_buffer_updates(models[0], buffer_updates, named_buffers=main_named_buffers)
|
| 363 |
+
for model in models[1:]:
|
| 364 |
+
apply_buffer_updates(model, buffer_updates)
|
| 365 |
+
return loaded
|
| 366 |
+
|
| 367 |
+
def _update_weights(
|
| 368 |
+
self,
|
| 369 |
+
weights: list[tuple[str, torch.Tensor]],
|
| 370 |
+
peft_config: dict,
|
| 371 |
+
base_sync_done: bool,
|
| 372 |
+
):
|
| 373 |
+
if peft_config and base_sync_done:
|
| 374 |
+
# Clone out of the receiver's reused IPC bucket buffer: add_lora keeps these tensors
|
| 375 |
+
# past this callback, so views into the freed/overwritten buffer crash later (#6454).
|
| 376 |
+
weights = {name: tensor.clone() for name, tensor in weights}
|
| 377 |
+
lora_request = TensorLoRARequest(
|
| 378 |
+
lora_name=VLLM_LORA_NAME,
|
| 379 |
+
lora_int_id=VLLM_LORA_INT_ID,
|
| 380 |
+
lora_path=VLLM_LORA_PATH,
|
| 381 |
+
peft_config=peft_config,
|
| 382 |
+
lora_tensors=weights,
|
| 383 |
+
)
|
| 384 |
+
self.add_lora(lora_request)
|
| 385 |
+
logger.info(f"vLLM load weights, loaded_params: {len(weights)}")
|
| 386 |
+
else:
|
| 387 |
+
param_updates, buffer_updates, named_buffers = split_buffer_updates(self.model_runner.model, weights)
|
| 388 |
+
# Add the FP8 related logic here as sharding manager has been deprecated.
|
| 389 |
+
# Check if FP8 quantization is enabled and apply appropriate weight loading
|
| 390 |
+
if is_quantized_model(self.model_runner.vllm_config):
|
| 391 |
+
logger.info(f"FP8 model detected (async): {self.model_runner.vllm_config.quant_config}")
|
| 392 |
+
# Convert bf16 weights to fp8 format before loading
|
| 393 |
+
loaded_params = load_quanted_weights(param_updates, self.model_runner) if param_updates else []
|
| 394 |
+
# Keep the draft model in sync when present.
|
| 395 |
+
if self._use_mtp_drafter_weight_sync() and param_updates:
|
| 396 |
+
load_quanted_weights(param_updates, self.model_runner, is_drafter=True)
|
| 397 |
+
loaded_buffers = self._apply_buffer_updates_all_models(buffer_updates, named_buffers)
|
| 398 |
+
logger.info(
|
| 399 |
+
f"FP8 weights loaded (async), loaded_params: {len(loaded_params)}, loaded_buffers: {loaded_buffers}"
|
| 400 |
+
)
|
| 401 |
+
else:
|
| 402 |
+
if param_updates:
|
| 403 |
+
for model in self._iter_all_models():
|
| 404 |
+
if peft_config is None:
|
| 405 |
+
model.load_weights(param_updates)
|
| 406 |
+
else:
|
| 407 |
+
names = {n for n, _ in model.named_parameters(remove_duplicate=False)}
|
| 408 |
+
names.update(n for n, _ in model.named_buffers())
|
| 409 |
+
model.load_weights((resolve_weight_name(model, n, names), t) for n, t in param_updates)
|
| 410 |
+
loaded_buffers = self._apply_buffer_updates_all_models(buffer_updates, named_buffers)
|
| 411 |
+
logger.info(
|
| 412 |
+
f"Loading standard weights (non-FP8, async), "
|
| 413 |
+
f"loaded_params: {len(param_updates)}, loaded_buffers: {loaded_buffers}"
|
| 414 |
+
)
|
| 415 |
+
|
| 416 |
+
def _get_zmq_handle(self) -> str:
|
| 417 |
+
"""Get ZMQ handle for communication.
|
| 418 |
+
|
| 419 |
+
Uses Ray job id + replica_rank + rollout-local rank to match the sender
|
| 420 |
+
side and avoid cross-job collisions on shared hosts.
|
| 421 |
+
In PD mode, each engine actor's local ranks start at 0; the optional
|
| 422 |
+
VERL_ZMQ_BASE_TRAINER_RANK offset maps them back to trainer ranks.
|
| 423 |
+
"""
|
| 424 |
+
replica_rank = os.environ.get("VERL_REPLICA_RANK", "0")
|
| 425 |
+
job_id = os.environ.get("VERL_RAY_JOB_ID", "0")
|
| 426 |
+
vllm_config = getattr(self.model_runner, "vllm_config", None)
|
| 427 |
+
parallel_config = getattr(vllm_config, "parallel_config", None)
|
| 428 |
+
local_rank = _resolve_vllm_weight_sync_local_rank(self.local_rank, parallel_config)
|
| 429 |
+
trainer_rank_base = os.environ.get("VERL_ZMQ_BASE_TRAINER_RANK")
|
| 430 |
+
trainer_rank = int(trainer_rank_base) + local_rank if trainer_rank_base is not None else local_rank
|
| 431 |
+
return f"ipc:///tmp/rl-colocate-zmq-{job_id}-replica-{replica_rank}-rank-{trainer_rank}.sock"
|
| 432 |
+
|
| 433 |
+
def update_verl_delta_weights(self, update_info: dict) -> None:
|
| 434 |
+
"""Add this worker's IPC endpoint and forward the delta update to vLLM."""
|
| 435 |
+
worker_update_info = dict(update_info)
|
| 436 |
+
worker_update_info["zmq_handle"] = self._get_zmq_handle()
|
| 437 |
+
self.update_weights(worker_update_info)
|
| 438 |
+
|
| 439 |
+
|
| 440 |
+
class SuppressSignalInThread:
|
| 441 |
+
def __enter__(self):
|
| 442 |
+
self.original_signal = signal.signal
|
| 443 |
+
|
| 444 |
+
def no_op_signal(sig, action):
|
| 445 |
+
if threading.current_thread() is not threading.main_thread():
|
| 446 |
+
print(f"Ignored signal {sig} in thread {threading.current_thread().name}")
|
| 447 |
+
return
|
| 448 |
+
return self.original_signal(sig, action)
|
| 449 |
+
|
| 450 |
+
signal.signal = no_op_signal
|
| 451 |
+
return self
|
| 452 |
+
|
| 453 |
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
| 454 |
+
signal.signal = self.original_signal
|
| 455 |
+
|
| 456 |
+
|
| 457 |
+
@functools.lru_cache(maxsize=1)
|
| 458 |
+
def _optional_bool_vllm_args() -> set[str]:
|
| 459 |
+
"""Return boolean engine fields for which omitting False changes semantics.
|
| 460 |
+
|
| 461 |
+
For such fields an omitted flag leaves the None default, which vLLM can
|
| 462 |
+
resolve to True at engine-config time (e.g. `enable_prefix_caching`), so
|
| 463 |
+
an explicit False must be serialized as `--no-<flag>` instead of being
|
| 464 |
+
dropped. Some vLLM versions annotate delayed-default fields as plain bool
|
| 465 |
+
despite a None default (e.g. enable_flashinfer_autotune in 0.26). Plain
|
| 466 |
+
bool fields defaulting to True also require an explicit negative flag.
|
| 467 |
+
"""
|
| 468 |
+
from vllm.engine.arg_utils import AsyncEngineArgs
|
| 469 |
+
|
| 470 |
+
return {
|
| 471 |
+
f.name
|
| 472 |
+
for f in dataclasses.fields(AsyncEngineArgs)
|
| 473 |
+
if set(get_args(f.type)) == {bool, type(None)} or (f.type is bool and (f.default is None or f.default is True))
|
| 474 |
+
}
|
| 475 |
+
|
| 476 |
+
|
| 477 |
+
def build_cli_args_from_config(config: dict[str, Any]) -> list[str]:
|
| 478 |
+
"""
|
| 479 |
+
Convert a config dictionary to CLI arguments for vLLM server.
|
| 480 |
+
|
| 481 |
+
Handles different value types appropriately:
|
| 482 |
+
- None: skipped
|
| 483 |
+
- bool True: adds '--key'
|
| 484 |
+
- bool False: adds '--no-key' for optional boolean engine args or plain
|
| 485 |
+
boolean engine args defaulting to None/True, otherwise skipped
|
| 486 |
+
- list: expands to '--key item1 item2 ...'
|
| 487 |
+
- empty list: skipped (vLLM uses nargs="+" which requires at least one value)
|
| 488 |
+
- dict: JSON serialized
|
| 489 |
+
- other: string converted
|
| 490 |
+
|
| 491 |
+
Args:
|
| 492 |
+
config: Dictionary of configuration key-value pairs
|
| 493 |
+
|
| 494 |
+
Returns:
|
| 495 |
+
List of CLI argument strings
|
| 496 |
+
"""
|
| 497 |
+
cli_args = []
|
| 498 |
+
for k, v in config.items():
|
| 499 |
+
if v is None:
|
| 500 |
+
continue
|
| 501 |
+
if isinstance(v, bool):
|
| 502 |
+
if v:
|
| 503 |
+
cli_args.append(f"--{k}")
|
| 504 |
+
elif k.replace("-", "_") in _optional_bool_vllm_args():
|
| 505 |
+
# Omission may preserve True or resolve a delayed None to True.
|
| 506 |
+
cli_args.append(f"--no-{k}")
|
| 507 |
+
elif isinstance(v, list):
|
| 508 |
+
if not v:
|
| 509 |
+
# Skip empty lists - vLLM uses nargs="+" which requires at least one value
|
| 510 |
+
continue
|
| 511 |
+
# Lists need to be expanded as multiple separate arguments
|
| 512 |
+
# e.g., --cuda-graph-sizes 1 2 4 8 becomes ['--cuda-graph-sizes', '1', '2', '4', '8']
|
| 513 |
+
cli_args.append(f"--{k}")
|
| 514 |
+
cli_args.extend([str(item) for item in v])
|
| 515 |
+
else:
|
| 516 |
+
cli_args.append(f"--{k}")
|
| 517 |
+
# Use json.dumps for dict to ensure valid JSON format
|
| 518 |
+
cli_args.append(json.dumps(v) if isinstance(v, dict) else str(v))
|
| 519 |
+
return cli_args
|
| 520 |
+
|
| 521 |
+
|
| 522 |
+
def build_mtp_speculative_config(
|
| 523 |
+
method: str, num_speculative_tokens: int, engine_speculative_config: Any = None
|
| 524 |
+
) -> dict[str, Any]:
|
| 525 |
+
"""Build vLLM's MTP speculative config, applying rollout engine overrides."""
|
| 526 |
+
if engine_speculative_config is None:
|
| 527 |
+
engine_speculative_config = {}
|
| 528 |
+
if isinstance(engine_speculative_config, str):
|
| 529 |
+
engine_speculative_config = json.loads(engine_speculative_config)
|
| 530 |
+
if not isinstance(engine_speculative_config, Mapping):
|
| 531 |
+
raise TypeError("rollout.engine_kwargs.vllm.speculative_config must be a mapping when MTP rollout is enabled")
|
| 532 |
+
|
| 533 |
+
return {
|
| 534 |
+
"method": method,
|
| 535 |
+
"num_speculative_tokens": num_speculative_tokens,
|
| 536 |
+
**{key: val for key, val in engine_speculative_config.items() if val is not None},
|
| 537 |
+
}
|
| 538 |
+
|
| 539 |
+
|
| 540 |
+
def extract_prompt_logprobs(output: RequestOutput, num_prompt_logprobs: Optional[int], result_dict: dict[str, list]):
|
| 541 |
+
"""Extract prompt log probabilities from generation output."""
|
| 542 |
+
if num_prompt_logprobs is None:
|
| 543 |
+
return
|
| 544 |
+
|
| 545 |
+
prompt_logprobs_ls, prompt_ids_ls = [], []
|
| 546 |
+
# NOTE: logprob of first prompt token is None.
|
| 547 |
+
for logprobs_dict in output.prompt_logprobs[1:]:
|
| 548 |
+
if num_prompt_logprobs == 0:
|
| 549 |
+
token_id_str = list(logprobs_dict.keys())[0]
|
| 550 |
+
logprob = logprobs_dict[token_id_str].logprob
|
| 551 |
+
prompt_logprobs_ls.append([logprob])
|
| 552 |
+
prompt_ids_ls.append([int(token_id_str)])
|
| 553 |
+
else:
|
| 554 |
+
prompt_ids = [None] * num_prompt_logprobs
|
| 555 |
+
prompt_logprobs = [None] * num_prompt_logprobs
|
| 556 |
+
# We get either top-k logprobs or top-k plus the sampled logprob (if sampled token is not in top-k)
|
| 557 |
+
assert len(logprobs_dict) in [num_prompt_logprobs, num_prompt_logprobs + 1], len(logprobs_dict)
|
| 558 |
+
for token_id_str, token_logprob in logprobs_dict.items():
|
| 559 |
+
rank = token_logprob.rank
|
| 560 |
+
if rank > num_prompt_logprobs:
|
| 561 |
+
continue # the sampled token is not in the top-k
|
| 562 |
+
logprob = token_logprob.logprob
|
| 563 |
+
prompt_ids[rank - 1] = int(token_id_str)
|
| 564 |
+
prompt_logprobs[rank - 1] = logprob
|
| 565 |
+
prompt_logprobs_ls.append(prompt_logprobs)
|
| 566 |
+
prompt_ids_ls.append(prompt_ids)
|
| 567 |
+
|
| 568 |
+
# NOTE: pad a dummy prompt logprob for last prompt token.
|
| 569 |
+
prompt_logprobs_ls.append([0.0] * max(num_prompt_logprobs, 1))
|
| 570 |
+
prompt_ids_ls.append([0] * max(num_prompt_logprobs, 1))
|
| 571 |
+
|
| 572 |
+
result_dict["prompt_ids"] = prompt_ids_ls
|
| 573 |
+
result_dict["prompt_logprobs"] = prompt_logprobs_ls
|
repro/vendor/verl/workers/rollout/vllm_rollout/vllm_async_server.py
ADDED
|
@@ -0,0 +1,1487 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2024 Bytedance Ltd. and/or its affiliates
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
import argparse
|
| 15 |
+
import asyncio
|
| 16 |
+
import inspect
|
| 17 |
+
import json
|
| 18 |
+
import logging
|
| 19 |
+
import os
|
| 20 |
+
import time
|
| 21 |
+
import uuid
|
| 22 |
+
from collections.abc import Mapping
|
| 23 |
+
from pprint import pprint
|
| 24 |
+
from typing import Any, Callable, Optional
|
| 25 |
+
|
| 26 |
+
import ray
|
| 27 |
+
import vllm.entrypoints.cli.serve
|
| 28 |
+
from packaging import version
|
| 29 |
+
from ray.actor import ActorHandle
|
| 30 |
+
from vllm import SamplingParams
|
| 31 |
+
from vllm.engine.arg_utils import AsyncEngineArgs
|
| 32 |
+
from vllm.entrypoints.cli.serve import run_headless
|
| 33 |
+
from vllm.entrypoints.openai.api_server import build_app, init_app_state
|
| 34 |
+
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
|
| 35 |
+
from vllm.inputs import TokensPrompt
|
| 36 |
+
from vllm.lora.request import LoRARequest
|
| 37 |
+
from vllm.outputs import RequestOutput
|
| 38 |
+
from vllm.usage.usage_lib import UsageContext
|
| 39 |
+
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
| 40 |
+
from vllm.v1.engine.async_llm import AsyncLLM
|
| 41 |
+
|
| 42 |
+
from verl.plugin.platform import get_platform
|
| 43 |
+
from verl.utils.config import omega_conf_to_dataclass
|
| 44 |
+
from verl.utils.device import get_resource_name, get_visible_devices_keyword, is_torch_npu_available
|
| 45 |
+
from verl.utils.net_utils import get_free_port, is_valid_ipv6_address
|
| 46 |
+
from verl.utils.profiler import (
|
| 47 |
+
build_rollout_dist_profiler,
|
| 48 |
+
build_vllm_profiler_args,
|
| 49 |
+
relocate_rollout_traces,
|
| 50 |
+
rollout_profiler_global_ranks,
|
| 51 |
+
)
|
| 52 |
+
from verl.utils.tokenizer import normalize_token_ids
|
| 53 |
+
from verl.utils.tracking import RLInsightLogger
|
| 54 |
+
from verl.utils.vllm.vllm_quant_utils import apply_vllm_quant_patches
|
| 55 |
+
from verl.workers.config import HFModelConfig, RolloutConfig
|
| 56 |
+
from verl.workers.rollout.replica import RolloutMode, RolloutReplica, TokenOutput
|
| 57 |
+
from verl.workers.rollout.utils import (
|
| 58 |
+
get_max_position_embeddings,
|
| 59 |
+
get_vision_placeholder_token_ids,
|
| 60 |
+
qwen2_5_vl_dedup_image_tokens,
|
| 61 |
+
run_uvicorn,
|
| 62 |
+
)
|
| 63 |
+
from verl.workers.rollout.vllm_rollout.utils import (
|
| 64 |
+
VLLM_LORA_INT_ID,
|
| 65 |
+
VLLM_LORA_NAME,
|
| 66 |
+
VLLM_LORA_PATH,
|
| 67 |
+
SuppressSignalInThread,
|
| 68 |
+
build_cli_args_from_config,
|
| 69 |
+
build_mtp_speculative_config,
|
| 70 |
+
extract_prompt_logprobs,
|
| 71 |
+
get_vllm_max_lora_rank,
|
| 72 |
+
)
|
| 73 |
+
|
| 74 |
+
_VLLM_VERSION = version.parse(vllm.__version__)
|
| 75 |
+
|
| 76 |
+
# Max wait for admissions already past the submission gate to reach the engine.
|
| 77 |
+
_GATE_BARRIER_TIMEOUT_S = 60.0
|
| 78 |
+
|
| 79 |
+
if os.getenv("VERL_USE_GPT_OSS", "0") == "1":
|
| 80 |
+
get_encoding()
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
logger = logging.getLogger(__file__)
|
| 84 |
+
logger.setLevel(logging.INFO)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
class vLLMHttpServer:
|
| 88 |
+
"""vLLM http server in single node, this is equivalent to launch server with command line:
|
| 89 |
+
```
|
| 90 |
+
vllm serve --tensor-parallel-size=8 ...
|
| 91 |
+
```
|
| 92 |
+
"""
|
| 93 |
+
|
| 94 |
+
def __init__(
|
| 95 |
+
self,
|
| 96 |
+
config,
|
| 97 |
+
model_config,
|
| 98 |
+
rollout_mode: RolloutMode,
|
| 99 |
+
workers: list[ActorHandle],
|
| 100 |
+
replica_rank: int,
|
| 101 |
+
node_rank: int,
|
| 102 |
+
gpus_per_node: int,
|
| 103 |
+
nnodes: int,
|
| 104 |
+
cuda_visible_devices: str,
|
| 105 |
+
disaggregation_role: str = "null",
|
| 106 |
+
disaggregation_kv_transfer_config: Optional[dict] = None,
|
| 107 |
+
):
|
| 108 |
+
"""
|
| 109 |
+
Args:
|
| 110 |
+
config (RolloutConfig): full config.
|
| 111 |
+
model_config (HFModelConfig): model config.
|
| 112 |
+
rollout_mode (RolloutMode): rollout mode.
|
| 113 |
+
replica_rank (int): replica rank, a replica may contain multiple nodes.
|
| 114 |
+
node_rank (int): node rank.
|
| 115 |
+
gpus_per_node (int): number of gpus per node.
|
| 116 |
+
nnodes (int): number of nodes.
|
| 117 |
+
cuda_visible_devices (str): cuda visible devices.
|
| 118 |
+
disaggregation_role: PD role, or ``"null"`` for normal rollout.
|
| 119 |
+
disaggregation_kv_transfer_config: vLLM KVTransferConfig dict for PD.
|
| 120 |
+
"""
|
| 121 |
+
if disaggregation_role not in ("null", "prefill", "decode"):
|
| 122 |
+
raise ValueError(f"disaggregation_role must be 'null'|'prefill'|'decode', got {disaggregation_role!r}")
|
| 123 |
+
if disaggregation_role != "null" and disaggregation_kv_transfer_config is None:
|
| 124 |
+
raise ValueError(
|
| 125 |
+
f"disaggregation_role={disaggregation_role!r} requires disaggregation_kv_transfer_config to be set"
|
| 126 |
+
)
|
| 127 |
+
self._disaggregation_role = disaggregation_role
|
| 128 |
+
self._disaggregation_kv_transfer_config = disaggregation_kv_transfer_config
|
| 129 |
+
# Filled by vLLMPDReplica.set_pd_peer for prefill-side routing.
|
| 130 |
+
self._pd_decode_peers: list[ActorHandle] = []
|
| 131 |
+
self._pd_prefill_side_channel_port: Optional[int] = None
|
| 132 |
+
self._pd_prefill_engine_id: Optional[str] = None
|
| 133 |
+
self._pd_peer_idx: int = 0
|
| 134 |
+
|
| 135 |
+
os.environ[get_visible_devices_keyword()] = cuda_visible_devices
|
| 136 |
+
os.environ["VERL_REPLICA_RANK"] = str(replica_rank)
|
| 137 |
+
# Forward the Ray job id into the vLLM worker subprocess so the
|
| 138 |
+
# colocated weight-transfer IPC socket path is unique per Ray job.
|
| 139 |
+
# Without this, two concurrent verl jobs on the same node both bind
|
| 140 |
+
# the same /tmp/rl-colocate-zmq-replica-0-rank-0.sock and one fails
|
| 141 |
+
# with EADDRINUSE; a stale socket from a crashed run trips the same
|
| 142 |
+
# error on restart.
|
| 143 |
+
os.environ["VERL_RAY_JOB_ID"] = ray.get_runtime_context().get_job_id()
|
| 144 |
+
|
| 145 |
+
self.config = self._init_config(config)
|
| 146 |
+
self.model_config = self._init_model_config(model_config)
|
| 147 |
+
self._validate_configs()
|
| 148 |
+
|
| 149 |
+
if self.config.full_determinism:
|
| 150 |
+
from verl.workers.engine.utils import enable_full_determinism
|
| 151 |
+
|
| 152 |
+
rollout_seed = replica_rank + self.config.seed
|
| 153 |
+
enable_full_determinism(seed=rollout_seed)
|
| 154 |
+
os.environ["VERL_FULL_DETERMINISM"] = "1"
|
| 155 |
+
os.environ["VERL_SEED"] = str(rollout_seed)
|
| 156 |
+
os.environ["VLLM_BATCH_INVARIANT"] = "1"
|
| 157 |
+
|
| 158 |
+
self.rollout_mode = rollout_mode
|
| 159 |
+
self.workers = workers
|
| 160 |
+
|
| 161 |
+
self.replica_rank = replica_rank
|
| 162 |
+
self.node_rank = node_rank
|
| 163 |
+
self.gpus_per_node = gpus_per_node
|
| 164 |
+
self.nnodes = nnodes
|
| 165 |
+
# model weights version, set by ServerAdapter when update weights.
|
| 166 |
+
self.global_steps = None
|
| 167 |
+
self._warned_missing_spec_decode_stats = False
|
| 168 |
+
|
| 169 |
+
# vLLM's pause stops requests being scheduled but still accepts them, and a request
|
| 170 |
+
# admitted after the pause is invisible to the drain's liveness check.
|
| 171 |
+
self._submission_paused = False
|
| 172 |
+
self._admitting = 0
|
| 173 |
+
self._resume_event = asyncio.Event()
|
| 174 |
+
self._resume_event.set()
|
| 175 |
+
# Set by abort_all_requests(reject_request=True) when no resume is coming soon.
|
| 176 |
+
self._rejecting = False
|
| 177 |
+
|
| 178 |
+
# used for http server
|
| 179 |
+
self._server_address = ray.util.get_node_ip_address().strip("[]")
|
| 180 |
+
self._server_port = None
|
| 181 |
+
|
| 182 |
+
# used for controlling vllm server profiler
|
| 183 |
+
profiler_config = self.config.profiler
|
| 184 |
+
tool_config = None
|
| 185 |
+
if profiler_config is not None:
|
| 186 |
+
if profiler_config.tool in ["torch", "npu"]:
|
| 187 |
+
tool_config = omega_conf_to_dataclass((profiler_config.tool_config or {}).get(profiler_config.tool))
|
| 188 |
+
else:
|
| 189 |
+
logger.warning(f"agent loop only support torch and npu profiler, got {profiler_config.tool}")
|
| 190 |
+
profiler_config = None
|
| 191 |
+
# `ranks` in the rollout profiler config are global GPU ranks (as in the training roles);
|
| 192 |
+
# map them to the replica that owns them so e.g. ranks=[0, 8] with tp=8 profiles the replicas
|
| 193 |
+
# holding global ranks 0 and 8 (replicas 0 and 1), not replica indices 0 and 8.
|
| 194 |
+
self.replica_world_size = (
|
| 195 |
+
self.config.tensor_model_parallel_size
|
| 196 |
+
* self.config.data_parallel_size
|
| 197 |
+
* self.config.pipeline_model_parallel_size
|
| 198 |
+
)
|
| 199 |
+
self.profiler_controller = build_rollout_dist_profiler(
|
| 200 |
+
self.replica_rank, self.replica_world_size, config=profiler_config, tool_config=tool_config
|
| 201 |
+
)
|
| 202 |
+
# A tp>1 engine profiles its whole replica, but the user asked for specific global GPU ranks;
|
| 203 |
+
# keep only those when relocating so ranks=[0, 8] yields exactly GPU 0 and 8, not their tp-mates.
|
| 204 |
+
self.profiler_keep_global_ranks = rollout_profiler_global_ranks(profiler_config)
|
| 205 |
+
|
| 206 |
+
# used for data parallel: --data-parallel-address, --data-parallel-rpc-port
|
| 207 |
+
if self.node_rank == 0:
|
| 208 |
+
self._master_address = self._server_address
|
| 209 |
+
# used for torch.distributed.init_process_group
|
| 210 |
+
self._master_port, self._master_sock = get_free_port(self._server_address, with_alive_sock=True)
|
| 211 |
+
# used for data parallel: --data-parallel-address, --data-parallel-rpc-port
|
| 212 |
+
self._dp_rpc_port, self._dp_rpc_sock = get_free_port(self._server_address, with_alive_sock=True)
|
| 213 |
+
self._dp_master_port, self._dp_master_sock = get_free_port(self._server_address, with_alive_sock=True)
|
| 214 |
+
else:
|
| 215 |
+
self._master_address = None
|
| 216 |
+
self._master_port = None
|
| 217 |
+
self._dp_rpc_port = None
|
| 218 |
+
self._dp_master_port = None
|
| 219 |
+
|
| 220 |
+
self._post_init(cuda_visible_devices)
|
| 221 |
+
|
| 222 |
+
def get_master_address(self):
|
| 223 |
+
"""Get master address and port for data parallel.
|
| 224 |
+
Returns:
|
| 225 |
+
tuple: (master_address, master_port, dp_rpc_port)
|
| 226 |
+
"""
|
| 227 |
+
return self._master_address, self._master_port, self._dp_rpc_port
|
| 228 |
+
|
| 229 |
+
def get_server_address(self):
|
| 230 |
+
"""Get http server address and port."""
|
| 231 |
+
assert self._server_port is not None, "http server is not launched, port is None"
|
| 232 |
+
return self._server_address, self._server_port
|
| 233 |
+
|
| 234 |
+
def get_rollout_config(self):
|
| 235 |
+
"""Get the RolloutConfig (e.g. max_num_seqs, max_model_len).
|
| 236 |
+
|
| 237 |
+
Lets external routers fetch server-side config (vLLM doesn't expose
|
| 238 |
+
these on /metrics) via the same handler-getter pattern as
|
| 239 |
+
``get_server_address``.
|
| 240 |
+
"""
|
| 241 |
+
return self.config
|
| 242 |
+
|
| 243 |
+
@property
|
| 244 |
+
def lora_as_adapter(self) -> bool:
|
| 245 |
+
return (
|
| 246 |
+
self.model_config.lora_rank > 0 or self.model_config.lora.get("rank", 0) > 0
|
| 247 |
+
) and not self.model_config.lora.get("merge", False)
|
| 248 |
+
|
| 249 |
+
async def collective_rpc(
|
| 250 |
+
self,
|
| 251 |
+
method: str | Callable,
|
| 252 |
+
timeout: float | None = None,
|
| 253 |
+
args: tuple = (),
|
| 254 |
+
kwargs: dict[str, Any] | None = None,
|
| 255 |
+
):
|
| 256 |
+
await self.engine.collective_rpc(
|
| 257 |
+
method=method,
|
| 258 |
+
timeout=timeout,
|
| 259 |
+
args=args,
|
| 260 |
+
kwargs=kwargs,
|
| 261 |
+
)
|
| 262 |
+
|
| 263 |
+
async def set_pd_peer(
|
| 264 |
+
self,
|
| 265 |
+
decode_peers: list,
|
| 266 |
+
prefill_side_channel_port: int,
|
| 267 |
+
prefill_engine_id: str,
|
| 268 |
+
) -> None:
|
| 269 |
+
assert self._disaggregation_role == "prefill", (
|
| 270 |
+
f"set_pd_peer must be called on the prefill server (got role={self._disaggregation_role!r})"
|
| 271 |
+
)
|
| 272 |
+
assert isinstance(decode_peers, list) and decode_peers, "decode_peers must be a non-empty list"
|
| 273 |
+
self._pd_decode_peers = list(decode_peers)
|
| 274 |
+
self._pd_prefill_side_channel_port = prefill_side_channel_port
|
| 275 |
+
self._pd_prefill_engine_id = prefill_engine_id
|
| 276 |
+
|
| 277 |
+
async def launch_server(self, master_address: str = None, master_port: int = None, dp_rpc_port: int = None):
|
| 278 |
+
if self.node_rank != 0:
|
| 279 |
+
assert master_address and master_port and dp_rpc_port, (
|
| 280 |
+
"non-master node should provide master_address, master_port and dp_rpc_port"
|
| 281 |
+
)
|
| 282 |
+
self._master_address = master_address
|
| 283 |
+
self._master_port = master_port
|
| 284 |
+
self._dp_rpc_port = dp_rpc_port
|
| 285 |
+
|
| 286 |
+
# 1. setup vllm serve cli args
|
| 287 |
+
engine_kwargs = self.config.get("engine_kwargs", {}).get(self._get_engine_kwargs_key(), {}) or {}
|
| 288 |
+
engine_kwargs = {key: val for key, val in engine_kwargs.items() if val is not None}
|
| 289 |
+
if self.config.get("limit_images", None): # support for multi-image data
|
| 290 |
+
engine_kwargs["limit_mm_per_prompt"] = {"image": self.config.get("limit_images")}
|
| 291 |
+
|
| 292 |
+
self._preprocess_engine_kwargs(engine_kwargs)
|
| 293 |
+
|
| 294 |
+
# Override default generation config from hugging face model config,
|
| 295 |
+
# user can still override them by passing kwargs in each request.
|
| 296 |
+
override_generation_config = self._get_override_generation_config()
|
| 297 |
+
logger.info(f"override_generation_config: {override_generation_config}")
|
| 298 |
+
|
| 299 |
+
logger.info(f"enable_sleep_mode: {self.config.enable_sleep_mode}")
|
| 300 |
+
if not self.config.enable_sleep_mode:
|
| 301 |
+
from verl.utils.device import set_expandable_segments
|
| 302 |
+
|
| 303 |
+
set_expandable_segments(True)
|
| 304 |
+
|
| 305 |
+
quantization, hf_overrides = self._apply_quantization()
|
| 306 |
+
|
| 307 |
+
compilation_config = engine_kwargs.pop("compilation_config", None) or {}
|
| 308 |
+
if isinstance(compilation_config, str):
|
| 309 |
+
compilation_config = json.loads(compilation_config)
|
| 310 |
+
compilation_config.setdefault("cudagraph_mode", "FULL_AND_PIECEWISE")
|
| 311 |
+
|
| 312 |
+
# FULL cuda graph is not yet supported with DCP, downgrade to PIECEWISE
|
| 313 |
+
dcp_size = engine_kwargs.get("decode_context_parallel_size", 1) or 1
|
| 314 |
+
if dcp_size > 1 and compilation_config["cudagraph_mode"] == "FULL_AND_PIECEWISE":
|
| 315 |
+
logger.warning(
|
| 316 |
+
"FULL cuda graph is not supported with DCP (decode_context_parallel_size=%d), "
|
| 317 |
+
"downgrading cudagraph_mode to PIECEWISE.",
|
| 318 |
+
dcp_size,
|
| 319 |
+
)
|
| 320 |
+
compilation_config["cudagraph_mode"] = "PIECEWISE"
|
| 321 |
+
if self.config.cudagraph_capture_sizes:
|
| 322 |
+
compilation_config["cudagraph_capture_sizes"] = self.config.cudagraph_capture_sizes
|
| 323 |
+
|
| 324 |
+
compilation_config = json.dumps(compilation_config)
|
| 325 |
+
args = {
|
| 326 |
+
"dtype": self.config.dtype,
|
| 327 |
+
"load_format": self.config.load_format,
|
| 328 |
+
"skip_tokenizer_init": False,
|
| 329 |
+
"distributed_executor_backend": "mp",
|
| 330 |
+
"worker_extension_cls": self._get_worker_extension_cls(),
|
| 331 |
+
"trust_remote_code": self.model_config.trust_remote_code,
|
| 332 |
+
"max_model_len": self.config.max_model_len,
|
| 333 |
+
"max_num_seqs": self.config.max_num_seqs,
|
| 334 |
+
"enable_chunked_prefill": self.config.enable_chunked_prefill,
|
| 335 |
+
"max_num_batched_tokens": self.config.max_num_batched_tokens,
|
| 336 |
+
"enable_prefix_caching": self.config.enable_prefix_caching,
|
| 337 |
+
"enable_sleep_mode": self.config.enable_sleep_mode,
|
| 338 |
+
"logprobs_mode": self.config.logprobs_mode,
|
| 339 |
+
"enforce_eager": self.config.enforce_eager,
|
| 340 |
+
"gpu_memory_utilization": self.config.gpu_memory_utilization,
|
| 341 |
+
"disable_log_stats": self.config.disable_log_stats,
|
| 342 |
+
"tensor_parallel_size": self.config.tensor_model_parallel_size,
|
| 343 |
+
"seed": self.replica_rank + self.config.seed,
|
| 344 |
+
"override_generation_config": json.dumps(override_generation_config),
|
| 345 |
+
"quantization": quantization,
|
| 346 |
+
"hf_overrides": hf_overrides,
|
| 347 |
+
"scheduling_policy": self.config.scheduling_policy,
|
| 348 |
+
"compilation_config": compilation_config,
|
| 349 |
+
**engine_kwargs,
|
| 350 |
+
}
|
| 351 |
+
|
| 352 |
+
# update profiler args, only on the replica that will actually be profiled: configuring
|
| 353 |
+
# the engine profiler everywhere makes every replica log that profiling is enabled while
|
| 354 |
+
# only the selected one is ever started.
|
| 355 |
+
if self._should_profile():
|
| 356 |
+
# vLLM >= 0.13.0 takes the profiler config as a CLI arg and warns about the legacy
|
| 357 |
+
# VLLM_TORCH_PROFILER_* environment variables it no longer reads.
|
| 358 |
+
use_cli_args = _VLLM_VERSION >= version.parse("0.13.0")
|
| 359 |
+
profiler_args = build_vllm_profiler_args(
|
| 360 |
+
self.profiler_controller.config,
|
| 361 |
+
self.profiler_controller.tool_config,
|
| 362 |
+
self.replica_rank,
|
| 363 |
+
legacy_env=not use_cli_args,
|
| 364 |
+
)
|
| 365 |
+
if use_cli_args:
|
| 366 |
+
args.update(profiler_args)
|
| 367 |
+
|
| 368 |
+
if self.config.prometheus.enable:
|
| 369 |
+
if self.config.prometheus.served_model_name:
|
| 370 |
+
# Extract model name from path if it's a full path
|
| 371 |
+
served_model_name = self.config.prometheus.served_model_name
|
| 372 |
+
if "/" in served_model_name:
|
| 373 |
+
# If it's a full path, extract the last part as model name
|
| 374 |
+
served_model_name = served_model_name.split("/")[-1]
|
| 375 |
+
args["served_model_name"] = served_model_name
|
| 376 |
+
|
| 377 |
+
if self.config.mtp is not None and self.config.mtp.enable and self.config.mtp.enable_rollout:
|
| 378 |
+
args["speculative_config"] = build_mtp_speculative_config(
|
| 379 |
+
self.config.mtp.method,
|
| 380 |
+
self.config.mtp.num_speculative_tokens,
|
| 381 |
+
args.get("speculative_config"),
|
| 382 |
+
)
|
| 383 |
+
|
| 384 |
+
if self.config.data_parallel_size > 1:
|
| 385 |
+
assert self.gpus_per_node % self.config.tensor_model_parallel_size == 0, (
|
| 386 |
+
"gpus_per_node should be divisible by tensor_model_parallel_size"
|
| 387 |
+
)
|
| 388 |
+
data_parallel_size_local = self.gpus_per_node // self.config.tensor_model_parallel_size
|
| 389 |
+
assert len(self.workers) == data_parallel_size_local * self.config.tensor_model_parallel_size, (
|
| 390 |
+
f"num workers ({len(self.workers)}) should be equal to "
|
| 391 |
+
f"dp_size_local ({data_parallel_size_local}) * tp_size ({self.config.tensor_model_parallel_size})"
|
| 392 |
+
)
|
| 393 |
+
dp_args = {
|
| 394 |
+
"data_parallel_size": self.config.data_parallel_size,
|
| 395 |
+
"data_parallel_size_local": data_parallel_size_local,
|
| 396 |
+
"data_parallel_start_rank": self.node_rank * data_parallel_size_local,
|
| 397 |
+
"data_parallel_address": self._master_address,
|
| 398 |
+
"data_parallel_rpc_port": self._dp_rpc_port,
|
| 399 |
+
}
|
| 400 |
+
args.update(dp_args)
|
| 401 |
+
|
| 402 |
+
args.update({"enable_expert_parallel": self.config.expert_parallel_size > 1})
|
| 403 |
+
|
| 404 |
+
# used for torch.distributed.init_process_group
|
| 405 |
+
if self.nnodes > 1:
|
| 406 |
+
args.update(
|
| 407 |
+
{
|
| 408 |
+
"master_addr": self._master_address,
|
| 409 |
+
"master_port": self._master_port,
|
| 410 |
+
"node_rank": self.node_rank,
|
| 411 |
+
"nnodes": self.nnodes,
|
| 412 |
+
"data_parallel_address": self._master_address,
|
| 413 |
+
"data_parallel_rpc_port": self._dp_rpc_port,
|
| 414 |
+
}
|
| 415 |
+
)
|
| 416 |
+
|
| 417 |
+
# update lora-related args
|
| 418 |
+
lora_rank = self.model_config.lora.get("rank", 0)
|
| 419 |
+
if lora_rank <= 0:
|
| 420 |
+
lora_rank = (
|
| 421 |
+
self.model_config.lora_rank
|
| 422 |
+
) # FIXME: fallback to lora_rank for now, we should unify lora settings.
|
| 423 |
+
|
| 424 |
+
if self.model_config.lora.get("merge", False):
|
| 425 |
+
lora_rank = 0
|
| 426 |
+
|
| 427 |
+
if lora_rank > 0:
|
| 428 |
+
lora_args = {
|
| 429 |
+
"enable_lora": True,
|
| 430 |
+
"max_loras": 1,
|
| 431 |
+
"max_lora_rank": get_vllm_max_lora_rank(lora_rank),
|
| 432 |
+
}
|
| 433 |
+
if self.model_config.lora.get("fully_sharded_loras", False):
|
| 434 |
+
lora_args["fully_sharded_loras"] = True
|
| 435 |
+
args.update(lora_args)
|
| 436 |
+
|
| 437 |
+
if self.config.enable_rollout_routing_replay:
|
| 438 |
+
# R3 (Rollout Router Replay) relies on vLLM's ``enable_return_routed_experts``
|
| 439 |
+
# path (RoutedExpertsManager / RoutedExpertsCapturer), which is only correct
|
| 440 |
+
# for hybrid-attention MoE models (e.g. Qwen3.5, whose linear + full attention
|
| 441 |
+
# layout produces >1 KV-cache group) starting from vLLM 0.22.0. Earlier
|
| 442 |
+
# releases either lack the feature or under-size the routed-experts host
|
| 443 |
+
# buffer and crash with an IndexError. Fail fast with an actionable message
|
| 444 |
+
# instead of surfacing an opaque runtime error deep inside vLLM.
|
| 445 |
+
if _VLLM_VERSION < version.parse("0.22.0"):
|
| 446 |
+
raise RuntimeError(
|
| 447 |
+
"rollout.enable_rollout_routing_replay=True requires vLLM >= 0.22.0 "
|
| 448 |
+
f"(installed: {vllm.__version__}). Upgrade vLLM (e.g. `pip install -U "
|
| 449 |
+
"'vllm>=0.22.0'`) or disable enable_rollout_routing_replay."
|
| 450 |
+
)
|
| 451 |
+
args.update({"enable_return_routed_experts": True})
|
| 452 |
+
|
| 453 |
+
if self._disaggregation_role != "null":
|
| 454 |
+
args["kv_transfer_config"] = json.dumps(self._disaggregation_kv_transfer_config)
|
| 455 |
+
|
| 456 |
+
server_args = ["serve", self.model_config.local_path] + build_cli_args_from_config(args)
|
| 457 |
+
|
| 458 |
+
if self.replica_rank == 0:
|
| 459 |
+
pprint(server_args)
|
| 460 |
+
|
| 461 |
+
CMD_MODULES = self._get_cli_modules()
|
| 462 |
+
parser = FlexibleArgumentParser(description=self._get_cli_description())
|
| 463 |
+
subparsers = parser.add_subparsers(required=False, dest="subparser")
|
| 464 |
+
cmds = {}
|
| 465 |
+
for cmd_module in CMD_MODULES:
|
| 466 |
+
new_cmds = cmd_module.cmd_init()
|
| 467 |
+
for cmd in new_cmds:
|
| 468 |
+
cmd.subparser_init(subparsers).set_defaults(dispatch_function=cmd.cmd)
|
| 469 |
+
cmds[cmd.name] = cmd
|
| 470 |
+
server_args = parser.parse_args(args=server_args)
|
| 471 |
+
server_args.model = server_args.model_tag
|
| 472 |
+
if server_args.subparser in cmds:
|
| 473 |
+
cmds[server_args.subparser].validate(server_args)
|
| 474 |
+
|
| 475 |
+
# 3. launch server
|
| 476 |
+
if self.node_rank == 0:
|
| 477 |
+
await self.run_server(server_args)
|
| 478 |
+
else:
|
| 479 |
+
await self.run_headless(server_args)
|
| 480 |
+
|
| 481 |
+
async def run_server(self, args: argparse.Namespace):
|
| 482 |
+
engine_args = AsyncEngineArgs.from_cli_args(args)
|
| 483 |
+
usage_context = UsageContext.OPENAI_API_SERVER
|
| 484 |
+
vllm_config = engine_args.create_engine_config(usage_context=usage_context)
|
| 485 |
+
vllm_config.parallel_config.data_parallel_master_port = self._dp_master_port
|
| 486 |
+
|
| 487 |
+
fn_args = set(dict(inspect.signature(AsyncLLM.from_vllm_config).parameters).keys())
|
| 488 |
+
kwargs = {}
|
| 489 |
+
if "enable_log_requests" in fn_args:
|
| 490 |
+
kwargs["enable_log_requests"] = engine_args.enable_log_requests
|
| 491 |
+
if "disable_log_stats" in fn_args:
|
| 492 |
+
kwargs["disable_log_stats"] = engine_args.disable_log_stats
|
| 493 |
+
|
| 494 |
+
engine_client = AsyncLLM.from_vllm_config(vllm_config=vllm_config, usage_context=usage_context, **kwargs)
|
| 495 |
+
|
| 496 |
+
# Don't keep the dummy data in memory
|
| 497 |
+
await engine_client.reset_mm_cache()
|
| 498 |
+
# A sampled <|image_pad|>/<|video_pad|> has no image behind it, and every consumer of the
|
| 499 |
+
# sequence assumes it does. Mask them out with the OOV tail, so the policy cannot pick one.
|
| 500 |
+
await engine_client.collective_rpc(
|
| 501 |
+
method="monkey_patch_model",
|
| 502 |
+
kwargs={
|
| 503 |
+
"vocab_size": len(self.model_config.tokenizer),
|
| 504 |
+
"banned_token_ids": get_vision_placeholder_token_ids(self.model_config.processor),
|
| 505 |
+
},
|
| 506 |
+
)
|
| 507 |
+
|
| 508 |
+
build_app_sig = inspect.signature(build_app)
|
| 509 |
+
supported_tasks: tuple[Any, ...] = ()
|
| 510 |
+
build_app_kwargs: dict[str, Any] = {}
|
| 511 |
+
if "supported_tasks" in build_app_sig.parameters:
|
| 512 |
+
supported_tasks = await engine_client.get_supported_tasks()
|
| 513 |
+
build_app_kwargs["supported_tasks"] = supported_tasks
|
| 514 |
+
# vLLM >= 0.20.0 requires `model_config` to register pooling API routes
|
| 515 |
+
# (e.g. ``/classify``, ``/embed``); see
|
| 516 |
+
# ``register_pooling_api_routers`` in vllm/entrypoints/pooling/factories.py
|
| 517 |
+
# which short-circuits when ``model_config`` is ``None``.
|
| 518 |
+
if "model_config" in build_app_sig.parameters:
|
| 519 |
+
build_app_kwargs["model_config"] = engine_client.model_config
|
| 520 |
+
app = build_app(args, **build_app_kwargs)
|
| 521 |
+
|
| 522 |
+
init_app_sig = inspect.signature(init_app_state)
|
| 523 |
+
if "vllm_config" in init_app_sig.parameters:
|
| 524 |
+
await init_app_state(engine_client, vllm_config, app.state, args)
|
| 525 |
+
elif "supported_tasks" in init_app_sig.parameters:
|
| 526 |
+
await init_app_state(engine_client, app.state, args, supported_tasks)
|
| 527 |
+
else:
|
| 528 |
+
await init_app_state(engine_client, app.state, args)
|
| 529 |
+
if self.replica_rank == 0 and self.node_rank == 0:
|
| 530 |
+
logger.info(f"Initializing a V1 LLM engine with config: {vllm_config}")
|
| 531 |
+
|
| 532 |
+
self.engine = engine_client
|
| 533 |
+
self._server_port, self._server_task = await run_uvicorn(app, args, self._server_address)
|
| 534 |
+
|
| 535 |
+
async def run_headless(self, args: argparse.Namespace):
|
| 536 |
+
"""Run headless server in a separate thread."""
|
| 537 |
+
args.api_server_count = 0
|
| 538 |
+
|
| 539 |
+
def run_headless_wrapper():
|
| 540 |
+
with SuppressSignalInThread():
|
| 541 |
+
run_headless(args)
|
| 542 |
+
|
| 543 |
+
def on_run_headless_done(future: asyncio.Future):
|
| 544 |
+
try:
|
| 545 |
+
exc = future.exception()
|
| 546 |
+
if exc:
|
| 547 |
+
logger.exception(f"run_headless failed with exception: {exc}")
|
| 548 |
+
else:
|
| 549 |
+
logger.warning("run_headless completed successfully, but it's not expected.")
|
| 550 |
+
except Exception as e:
|
| 551 |
+
logger.exception(f"get result from run_headless failed: {e}")
|
| 552 |
+
finally:
|
| 553 |
+
os._exit(1)
|
| 554 |
+
|
| 555 |
+
self.task = asyncio.create_task(asyncio.to_thread(run_headless_wrapper))
|
| 556 |
+
self.task.add_done_callback(on_run_headless_done)
|
| 557 |
+
|
| 558 |
+
async def generate(
|
| 559 |
+
self,
|
| 560 |
+
prompt_ids: list[int],
|
| 561 |
+
sampling_params: dict[str, Any],
|
| 562 |
+
request_id: str,
|
| 563 |
+
image_data: Optional[list[Any]] = None,
|
| 564 |
+
video_data: Optional[list[Any]] = None,
|
| 565 |
+
audio_data: Optional[list[Any]] = None,
|
| 566 |
+
mm_processor_kwargs: Optional[dict[str, Any]] = None,
|
| 567 |
+
priority: int = 0,
|
| 568 |
+
kv_transfer_params: Optional[dict] = None,
|
| 569 |
+
) -> TokenOutput:
|
| 570 |
+
"""Generate sequence with token-in-token-out.
|
| 571 |
+
|
| 572 |
+
Args:
|
| 573 |
+
kv_transfer_params: vLLM KV-transfer payload for PD requests.
|
| 574 |
+
"""
|
| 575 |
+
if self._disaggregation_role == "prefill" and self._pd_decode_peers and kv_transfer_params is None:
|
| 576 |
+
return await self._pd_dispatch(
|
| 577 |
+
prompt_ids,
|
| 578 |
+
sampling_params,
|
| 579 |
+
request_id,
|
| 580 |
+
image_data=image_data,
|
| 581 |
+
video_data=video_data,
|
| 582 |
+
audio_data=audio_data,
|
| 583 |
+
mm_processor_kwargs=mm_processor_kwargs,
|
| 584 |
+
priority=priority,
|
| 585 |
+
)
|
| 586 |
+
|
| 587 |
+
prompt_ids = normalize_token_ids(prompt_ids)
|
| 588 |
+
|
| 589 |
+
# Calculate the maximum possible new tokens based on available context space
|
| 590 |
+
# This serves as a safety upper bound. vLLM v0.20+ rejects `max_tokens < 1`
|
| 591 |
+
# (see vllm.sampling_params.SamplingParams._verify_args), so we require at
|
| 592 |
+
# least one token of headroom to be able to generate at all.
|
| 593 |
+
max_possible_tokens = self.config.max_model_len - len(prompt_ids)
|
| 594 |
+
if max_possible_tokens < 1:
|
| 595 |
+
raise ValueError(
|
| 596 |
+
f"Prompt length ({len(prompt_ids)}) leaves no room to generate within the "
|
| 597 |
+
f"model's maximum context length ({self.config.max_model_len}); need at least "
|
| 598 |
+
f"1 token of headroom."
|
| 599 |
+
)
|
| 600 |
+
|
| 601 |
+
# Determine max_tokens from sampling_params or use configured response_length as default
|
| 602 |
+
if "max_tokens" in sampling_params:
|
| 603 |
+
max_tokens = sampling_params.pop("max_tokens")
|
| 604 |
+
elif "max_new_tokens" in sampling_params:
|
| 605 |
+
# support sglang-style 'max_new_tokens' param
|
| 606 |
+
max_tokens = sampling_params.pop("max_new_tokens")
|
| 607 |
+
else:
|
| 608 |
+
# Default to a calculation that considers configured lengths
|
| 609 |
+
# Cap max_tokens by response_length to ensure tensor alignment,
|
| 610 |
+
# and by remaining budget to prevent OOM in multi-turn rollouts.
|
| 611 |
+
max_tokens = min(
|
| 612 |
+
self.config.response_length, self.config.prompt_length + self.config.response_length - len(prompt_ids)
|
| 613 |
+
)
|
| 614 |
+
|
| 615 |
+
# Clamp max_tokens to the valid range [1, max_possible_tokens]. The lower bound
|
| 616 |
+
# is 1 because vLLM v0.20+ raises VLLMValidationError when max_tokens < 1.
|
| 617 |
+
max_tokens = max(1, min(max_tokens, max_possible_tokens))
|
| 618 |
+
|
| 619 |
+
assert 1 <= max_tokens <= max_possible_tokens, (
|
| 620 |
+
f"max_tokens {max_tokens} not in valid range [1, {max_possible_tokens}]"
|
| 621 |
+
)
|
| 622 |
+
sampling_params["logprobs"] = 0 if sampling_params.pop("logprobs", False) else None
|
| 623 |
+
sampling_params.setdefault("repetition_penalty", self.config.get("repetition_penalty", 1.0))
|
| 624 |
+
sampling_params.setdefault("ignore_eos", self.config.get("ignore_eos", False))
|
| 625 |
+
# Inject per-request seed for deterministic sampling when full_determinism is enabled.
|
| 626 |
+
if self.config.full_determinism:
|
| 627 |
+
sampling_params.setdefault("seed", self.replica_rank + self.config.seed)
|
| 628 |
+
|
| 629 |
+
if kv_transfer_params is not None:
|
| 630 |
+
extra_args = dict(sampling_params.pop("extra_args", None) or {})
|
| 631 |
+
extra_args["kv_transfer_params"] = kv_transfer_params
|
| 632 |
+
sampling_params["extra_args"] = extra_args
|
| 633 |
+
|
| 634 |
+
sampling_params = SamplingParams(max_tokens=max_tokens, **sampling_params)
|
| 635 |
+
prompt_ids = qwen2_5_vl_dedup_image_tokens(prompt_ids, self.model_config.processor)
|
| 636 |
+
multi_modal_data = {}
|
| 637 |
+
if image_data is not None:
|
| 638 |
+
multi_modal_data["image"] = image_data
|
| 639 |
+
if video_data is not None:
|
| 640 |
+
multi_modal_data["video"] = video_data
|
| 641 |
+
if audio_data is not None:
|
| 642 |
+
multi_modal_data["audio"] = audio_data
|
| 643 |
+
|
| 644 |
+
prompt_kwargs = {"prompt_token_ids": prompt_ids, "multi_modal_data": multi_modal_data}
|
| 645 |
+
if mm_processor_kwargs:
|
| 646 |
+
prompt_kwargs["mm_processor_kwargs"] = mm_processor_kwargs
|
| 647 |
+
try:
|
| 648 |
+
prompt = TokensPrompt(**prompt_kwargs)
|
| 649 |
+
except TypeError:
|
| 650 |
+
prompt = prompt_kwargs
|
| 651 |
+
|
| 652 |
+
# Add lora request
|
| 653 |
+
lora_request = None
|
| 654 |
+
if self.lora_as_adapter:
|
| 655 |
+
# Make sure we also check that the lora is already loaded in the engine
|
| 656 |
+
lora_loaded = VLLM_LORA_INT_ID in await self.engine.list_loras()
|
| 657 |
+
if lora_loaded:
|
| 658 |
+
lora_request = LoRARequest(
|
| 659 |
+
lora_name=VLLM_LORA_NAME, lora_int_id=VLLM_LORA_INT_ID, lora_path=VLLM_LORA_PATH
|
| 660 |
+
)
|
| 661 |
+
|
| 662 |
+
rejected = await self._park_until_admitted(request_id)
|
| 663 |
+
if rejected is not None:
|
| 664 |
+
return rejected
|
| 665 |
+
|
| 666 |
+
with RLInsightLogger.trace_state("vllm_generate", state_lane_id=f"replica_{self.replica_rank}"):
|
| 667 |
+
generator = self.engine.generate(
|
| 668 |
+
prompt=prompt,
|
| 669 |
+
sampling_params=sampling_params,
|
| 670 |
+
request_id=request_id,
|
| 671 |
+
lora_request=lora_request,
|
| 672 |
+
priority=priority,
|
| 673 |
+
)
|
| 674 |
+
|
| 675 |
+
# Get final response
|
| 676 |
+
final_res: Optional[RequestOutput] = None
|
| 677 |
+
admitted = False
|
| 678 |
+
try:
|
| 679 |
+
async for output in generator:
|
| 680 |
+
if not admitted:
|
| 681 |
+
admitted = True
|
| 682 |
+
self._admitting -= 1
|
| 683 |
+
final_res = output
|
| 684 |
+
finally:
|
| 685 |
+
if not admitted:
|
| 686 |
+
self._admitting -= 1
|
| 687 |
+
assert final_res is not None
|
| 688 |
+
|
| 689 |
+
extra_fields = {"global_steps": self.global_steps}
|
| 690 |
+
# Handle abort case: when the request is aborted by pause_generation(abort),
|
| 691 |
+
# outputs may be empty. Return empty results with stop_reason="aborted"
|
| 692 |
+
# instead of crashing with "IndexError: list index out of range".
|
| 693 |
+
if not final_res.outputs:
|
| 694 |
+
return TokenOutput(
|
| 695 |
+
token_ids=[],
|
| 696 |
+
log_probs=None,
|
| 697 |
+
routed_experts=None,
|
| 698 |
+
stop_reason="aborted",
|
| 699 |
+
extra_fields=extra_fields,
|
| 700 |
+
)
|
| 701 |
+
|
| 702 |
+
# Prefix-cache hit count for this request; consumers surface it as
|
| 703 |
+
# OpenAI usage.prompt_tokens_details.cached_tokens.
|
| 704 |
+
extra_fields["num_cached_tokens"] = getattr(final_res, "num_cached_tokens", None)
|
| 705 |
+
extract_prompt_logprobs(
|
| 706 |
+
output=final_res,
|
| 707 |
+
num_prompt_logprobs=sampling_params.prompt_logprobs,
|
| 708 |
+
result_dict=extra_fields,
|
| 709 |
+
)
|
| 710 |
+
token_ids = final_res.outputs[0].token_ids
|
| 711 |
+
log_probs = None
|
| 712 |
+
if sampling_params.logprobs is not None:
|
| 713 |
+
log_probs = [logprobs[token_ids[i]].logprob for i, logprobs in enumerate(final_res.outputs[0].logprobs)]
|
| 714 |
+
|
| 715 |
+
routed_experts = None
|
| 716 |
+
if self.config.enable_rollout_routing_replay:
|
| 717 |
+
routed_experts = final_res.outputs[0].routed_experts
|
| 718 |
+
|
| 719 |
+
# Determine stop reason from finish_reason
|
| 720 |
+
finish_reason = final_res.outputs[0].finish_reason
|
| 721 |
+
if finish_reason == "abort":
|
| 722 |
+
stop_reason = "aborted"
|
| 723 |
+
elif finish_reason in ("stop", "length"):
|
| 724 |
+
stop_reason = "completed"
|
| 725 |
+
else:
|
| 726 |
+
stop_reason = finish_reason # for more stop reason in the future
|
| 727 |
+
|
| 728 |
+
num_preempted = None
|
| 729 |
+
|
| 730 |
+
if hasattr(final_res.outputs[0], "num_preempted"):
|
| 731 |
+
num_preempted = final_res.outputs[0].num_preempted
|
| 732 |
+
|
| 733 |
+
response_kv_transfer_params = getattr(final_res, "kv_transfer_params", None)
|
| 734 |
+
if response_kv_transfer_params is not None:
|
| 735 |
+
extra_fields["kv_transfer_params"] = response_kv_transfer_params
|
| 736 |
+
|
| 737 |
+
# Re-key backend spec-decoding stats to the rollout-common names.
|
| 738 |
+
if self.config.mtp is not None and self.config.mtp.enable and self.config.mtp.enable_rollout:
|
| 739 |
+
spec_decode_stats = getattr(final_res.metrics, "request_spec_decode_stats", None)
|
| 740 |
+
if spec_decode_stats is None:
|
| 741 |
+
if not self._warned_missing_spec_decode_stats:
|
| 742 |
+
logger.warning(
|
| 743 |
+
"vLLM MTP rollout metrics do not include request_spec_decode_stats; "
|
| 744 |
+
"speculative decoding acceptance metrics will be skipped."
|
| 745 |
+
)
|
| 746 |
+
self._warned_missing_spec_decode_stats = True
|
| 747 |
+
else:
|
| 748 |
+
extra_fields["spec_num_draft_tokens"] = spec_decode_stats.num_draft_tokens
|
| 749 |
+
extra_fields["spec_num_accepted_tokens"] = spec_decode_stats.num_accepted_tokens
|
| 750 |
+
extra_fields["spec_num_verify_steps"] = spec_decode_stats.num_verify_steps
|
| 751 |
+
return TokenOutput(
|
| 752 |
+
token_ids=token_ids,
|
| 753 |
+
log_probs=log_probs,
|
| 754 |
+
routed_experts=routed_experts,
|
| 755 |
+
stop_reason=stop_reason,
|
| 756 |
+
num_preempted=num_preempted,
|
| 757 |
+
extra_fields=extra_fields,
|
| 758 |
+
)
|
| 759 |
+
|
| 760 |
+
def _select_decode_peer(self) -> ActorHandle:
|
| 761 |
+
"""Round-robin across decode peers."""
|
| 762 |
+
peer = self._pd_decode_peers[self._pd_peer_idx % len(self._pd_decode_peers)]
|
| 763 |
+
self._pd_peer_idx += 1
|
| 764 |
+
return peer
|
| 765 |
+
|
| 766 |
+
async def _pd_dispatch(
|
| 767 |
+
self,
|
| 768 |
+
prompt_ids: list[int],
|
| 769 |
+
sampling_params: dict[str, Any],
|
| 770 |
+
request_id: str,
|
| 771 |
+
image_data: Optional[list[Any]] = None,
|
| 772 |
+
video_data: Optional[list[Any]] = None,
|
| 773 |
+
audio_data: Optional[list[Any]] = None,
|
| 774 |
+
mm_processor_kwargs: Optional[dict[str, Any]] = None,
|
| 775 |
+
priority: int = 0,
|
| 776 |
+
) -> TokenOutput:
|
| 777 |
+
"""Run prefill locally, then decode on a selected peer."""
|
| 778 |
+
decode_peer = self._select_decode_peer()
|
| 779 |
+
connector = (self._disaggregation_kv_transfer_config or {}).get("kv_connector", "")
|
| 780 |
+
is_mooncake = connector == "MooncakeConnector"
|
| 781 |
+
|
| 782 |
+
# Prefill only materializes KV; discard its single generated token.
|
| 783 |
+
prefill_sp = dict(sampling_params)
|
| 784 |
+
prefill_sp.pop("max_tokens", None)
|
| 785 |
+
prefill_sp.pop("max_new_tokens", None)
|
| 786 |
+
prefill_sp["max_tokens"] = 1
|
| 787 |
+
transfer_id = uuid.uuid4().hex
|
| 788 |
+
prefill_kv_params = {
|
| 789 |
+
"do_remote_decode": True,
|
| 790 |
+
"do_remote_prefill": False,
|
| 791 |
+
"transfer_id": transfer_id,
|
| 792 |
+
}
|
| 793 |
+
|
| 794 |
+
prefill_out = await self.generate(
|
| 795 |
+
prompt_ids,
|
| 796 |
+
prefill_sp,
|
| 797 |
+
f"{request_id}_P",
|
| 798 |
+
image_data=image_data,
|
| 799 |
+
video_data=video_data,
|
| 800 |
+
audio_data=audio_data,
|
| 801 |
+
mm_processor_kwargs=mm_processor_kwargs,
|
| 802 |
+
priority=priority,
|
| 803 |
+
kv_transfer_params=prefill_kv_params,
|
| 804 |
+
)
|
| 805 |
+
if is_mooncake:
|
| 806 |
+
# Mooncake does not return decode params from the prefill leg.
|
| 807 |
+
decode_kv_params = {
|
| 808 |
+
"do_remote_decode": False,
|
| 809 |
+
"do_remote_prefill": True,
|
| 810 |
+
"remote_engine_id": self._pd_prefill_engine_id,
|
| 811 |
+
# Single-node PD uses Mooncake's local bootstrap address.
|
| 812 |
+
"remote_bootstrap_addr": f"http://127.0.0.1:{self._pd_prefill_side_channel_port}",
|
| 813 |
+
"transfer_id": transfer_id,
|
| 814 |
+
}
|
| 815 |
+
else:
|
| 816 |
+
decode_kv_params = prefill_out.extra_fields.get("kv_transfer_params")
|
| 817 |
+
if decode_kv_params is None:
|
| 818 |
+
raise RuntimeError(f"PD prefill leg returned no kv_transfer_params (request_id={request_id})")
|
| 819 |
+
|
| 820 |
+
return await decode_peer.generate.remote(
|
| 821 |
+
prompt_ids,
|
| 822 |
+
dict(sampling_params),
|
| 823 |
+
f"{request_id}_D",
|
| 824 |
+
image_data=image_data,
|
| 825 |
+
video_data=video_data,
|
| 826 |
+
audio_data=audio_data,
|
| 827 |
+
mm_processor_kwargs=mm_processor_kwargs,
|
| 828 |
+
priority=priority,
|
| 829 |
+
kv_transfer_params=decode_kv_params,
|
| 830 |
+
)
|
| 831 |
+
|
| 832 |
+
async def wake_up(self, tags: list[str] | None = None):
|
| 833 |
+
if self.node_rank != 0:
|
| 834 |
+
return
|
| 835 |
+
|
| 836 |
+
if self.rollout_mode == RolloutMode.HYBRID:
|
| 837 |
+
# engine.wake_up() broadcasts via the DP coordinator to ALL EngineCore
|
| 838 |
+
# processes across all DP shards (unlike collective_rpc which only reaches
|
| 839 |
+
# TP workers within a single shard).
|
| 840 |
+
await self.engine.wake_up(tags=tags or self._get_wake_up_tags())
|
| 841 |
+
await self.engine.reset_prefix_cache(reset_connector=True)
|
| 842 |
+
elif self.rollout_mode == RolloutMode.COLOCATED:
|
| 843 |
+
# Directly call engine to wake up without sync weights.
|
| 844 |
+
await self.engine.wake_up(tags=self._get_wake_up_tags())
|
| 845 |
+
# reset_connector=True drops any attached external KV store
|
| 846 |
+
# (e.g. MooncakeStoreConnector) whose entries were computed
|
| 847 |
+
# against the previous weights. No-op success when no connector
|
| 848 |
+
# is configured (vLLM scheduler treats it as such).
|
| 849 |
+
await self.engine.reset_prefix_cache(reset_connector=True)
|
| 850 |
+
elif self.rollout_mode == RolloutMode.STANDALONE:
|
| 851 |
+
logger.info("skip wake_up in standalone mode")
|
| 852 |
+
|
| 853 |
+
async def sleep(self):
|
| 854 |
+
if self.node_rank != 0 or not self.config.free_cache_engine:
|
| 855 |
+
return
|
| 856 |
+
|
| 857 |
+
if self.rollout_mode == RolloutMode.HYBRID:
|
| 858 |
+
await self._sleep_hybrid()
|
| 859 |
+
elif self.rollout_mode == RolloutMode.COLOCATED:
|
| 860 |
+
await self.engine.sleep(level=1)
|
| 861 |
+
elif self.rollout_mode == RolloutMode.STANDALONE:
|
| 862 |
+
logger.info("skip sleep in standalone mode")
|
| 863 |
+
|
| 864 |
+
async def clear_kv_cache(self):
|
| 865 |
+
if self.node_rank == 0:
|
| 866 |
+
# reset_connector=True drops any attached external KV store
|
| 867 |
+
# (e.g. MooncakeStoreConnector) whose entries were computed
|
| 868 |
+
# against the previous model weights. With no connector it
|
| 869 |
+
# is a no-op success, so we can pass it unconditionally.
|
| 870 |
+
await self.engine.reset_prefix_cache(reset_connector=True)
|
| 871 |
+
|
| 872 |
+
await self.engine.reset_mm_cache()
|
| 873 |
+
await self.engine.reset_encoder_cache()
|
| 874 |
+
|
| 875 |
+
async def release_kv_cache(self):
|
| 876 |
+
"""Free the kv_cache pool for the duration of a weight sync."""
|
| 877 |
+
# TODO: use the real release_kv_cache() method after vllm supports it (vllm#44890/46438)
|
| 878 |
+
if self.node_rank != 0 or not self.config.free_cache_engine:
|
| 879 |
+
return
|
| 880 |
+
if self.rollout_mode == RolloutMode.COLOCATED:
|
| 881 |
+
return
|
| 882 |
+
await self.engine.sleep(level=self._resolve_sleep_level())
|
| 883 |
+
await self.engine.wake_up(tags=["weights"])
|
| 884 |
+
|
| 885 |
+
async def resume_kv_cache(self):
|
| 886 |
+
"""Restore kv_cache GPU memory after a weight sync. Counterpart to release_kv_cache()."""
|
| 887 |
+
if self.node_rank != 0 or not self.config.free_cache_engine:
|
| 888 |
+
return
|
| 889 |
+
if self.rollout_mode == RolloutMode.COLOCATED:
|
| 890 |
+
return
|
| 891 |
+
await self.engine.wake_up(tags=["kv_cache"])
|
| 892 |
+
await self.engine.reset_prefix_cache(reset_connector=True)
|
| 893 |
+
|
| 894 |
+
def _should_profile(self) -> bool:
|
| 895 |
+
"""Whether this replica drives the engine profiler."""
|
| 896 |
+
return (
|
| 897 |
+
self.profiler_controller.check_enable()
|
| 898 |
+
and self.profiler_controller.check_this_rank()
|
| 899 |
+
and self.profiler_controller.is_discrete_mode()
|
| 900 |
+
)
|
| 901 |
+
|
| 902 |
+
async def start_profile(self, **kwargs):
|
| 903 |
+
if self.node_rank != 0:
|
| 904 |
+
return
|
| 905 |
+
if self._should_profile():
|
| 906 |
+
await self.engine.start_profile(**kwargs)
|
| 907 |
+
|
| 908 |
+
async def stop_profile(self):
|
| 909 |
+
if self.node_rank != 0:
|
| 910 |
+
return
|
| 911 |
+
if self._should_profile():
|
| 912 |
+
await self.engine.stop_profile()
|
| 913 |
+
# Relocate the engine's traces into save_path (when relocate_results is set) so the
|
| 914 |
+
# training worker's single end-of-run upload of the whole save_path picks them up. The
|
| 915 |
+
# rollout engine does not run the finish command itself: it shares save_path with the
|
| 916 |
+
# colocated training worker, so uploading here too would send the same directory twice.
|
| 917 |
+
relocate_rollout_traces(
|
| 918 |
+
self.profiler_controller.config,
|
| 919 |
+
self.replica_rank,
|
| 920 |
+
self.replica_world_size,
|
| 921 |
+
self.profiler_keep_global_ranks,
|
| 922 |
+
)
|
| 923 |
+
|
| 924 |
+
async def set_global_steps(self, global_steps: int):
|
| 925 |
+
"""Set the global steps of the model weights."""
|
| 926 |
+
self.global_steps = global_steps
|
| 927 |
+
|
| 928 |
+
async def wait_for_requests_to_drain(self):
|
| 929 |
+
await self.engine.wait_for_requests_to_drain()
|
| 930 |
+
|
| 931 |
+
async def abort_all_requests(self, reset_prefix_cache: bool = True, reject_request: bool = False) -> dict[str, Any]:
|
| 932 |
+
"""Abort all ongoing generation requests.
|
| 933 |
+
|
| 934 |
+
On vLLM >= 0.12.0, uses AsyncLLM.pause_generation() to abort in-flight
|
| 935 |
+
requests, drain, and clear caches. The engine remains paused after this
|
| 936 |
+
call — use resume_generation() to accept new requests (e.g. before
|
| 937 |
+
validation).
|
| 938 |
+
|
| 939 |
+
On vLLM < 0.12.0, manually aborts each request and resets prefix cache.
|
| 940 |
+
|
| 941 |
+
Args:
|
| 942 |
+
reset_prefix_cache: Clear the prefix/mm caches along with the pause.
|
| 943 |
+
reject_request: Fail requests that arrive while the gate is closed
|
| 944 |
+
instead of parking them. Pass True when this server is leaving
|
| 945 |
+
the load balancer with no resume_generation coming soon (a hybrid
|
| 946 |
+
replica handed back to training); parked requests would otherwise
|
| 947 |
+
wait until the replica returns to rotation. The flag is re-declared
|
| 948 |
+
by every abort and cleared by the matching resume, so a subsequent
|
| 949 |
+
plain abort (e.g. the one inside a weight sync) restores parking.
|
| 950 |
+
|
| 951 |
+
Returns:
|
| 952 |
+
dict[str, Any]: Dictionary containing:
|
| 953 |
+
- aborted_count: Number of requests aborted
|
| 954 |
+
- request_ids: List of aborted request IDs
|
| 955 |
+
"""
|
| 956 |
+
# Only node rank 0 owns AsyncLLM/self.engine. The remaining actors in a
|
| 957 |
+
# multi-node replica run vLLM's headless entry point, so there is no
|
| 958 |
+
# engine object to abort through on those actors.
|
| 959 |
+
if self.node_rank != 0:
|
| 960 |
+
return {"aborted_count": 0, "request_ids": []}
|
| 961 |
+
|
| 962 |
+
try:
|
| 963 |
+
# Close the gate first, then let admissions already past it land, so the pause
|
| 964 |
+
# below actually covers them. Closing the gate and declaring how late arrivals
|
| 965 |
+
# are handled happen together, so no request can observe one without the other.
|
| 966 |
+
self._submission_paused = True
|
| 967 |
+
self._rejecting = reject_request
|
| 968 |
+
self._resume_event.clear()
|
| 969 |
+
deadline = time.monotonic() + _GATE_BARRIER_TIMEOUT_S
|
| 970 |
+
while self._admitting > 0:
|
| 971 |
+
if time.monotonic() > deadline:
|
| 972 |
+
logger.warning(
|
| 973 |
+
"Submission gate barrier timed out with %d admission(s) in flight, proceeding",
|
| 974 |
+
self._admitting,
|
| 975 |
+
)
|
| 976 |
+
break
|
| 977 |
+
await asyncio.sleep(0.01)
|
| 978 |
+
|
| 979 |
+
# Snapshot request IDs before pausing for reporting
|
| 980 |
+
request_ids = list(self.engine.output_processor.request_states.keys())
|
| 981 |
+
|
| 982 |
+
# pause_generation with wait_for_inflight_requests=False will:
|
| 983 |
+
# 1. Set engine to paused state (new requests are accepted but not scheduled)
|
| 984 |
+
# 2. Abort all in-flight requests
|
| 985 |
+
# 3. Wait for requests to drain
|
| 986 |
+
# 4. Clear prefix and mm caches if clear_cache=True.
|
| 987 |
+
# EngineCore._reset_caches defaults reset_connector=True
|
| 988 |
+
# on this path, so any attached external KV store (e.g.
|
| 989 |
+
# MooncakeStoreConnector) is invalidated along with the
|
| 990 |
+
# local prefix cache — RL-correct hard-reset at every
|
| 991 |
+
# weight update boundary, no extra kwargs needed.
|
| 992 |
+
await self.engine.pause_generation(
|
| 993 |
+
wait_for_inflight_requests=False,
|
| 994 |
+
clear_cache=reset_prefix_cache,
|
| 995 |
+
)
|
| 996 |
+
|
| 997 |
+
logger.info(f"Aborted {len(request_ids)} requests: {request_ids}")
|
| 998 |
+
return {"aborted_count": len(request_ids), "request_ids": request_ids}
|
| 999 |
+
|
| 1000 |
+
except Exception:
|
| 1001 |
+
# Weight updates must not proceed unless every in-flight request was
|
| 1002 |
+
# actually aborted and the old-weight caches were cleared.
|
| 1003 |
+
logger.exception("Error aborting requests")
|
| 1004 |
+
raise
|
| 1005 |
+
|
| 1006 |
+
async def resume_engine_generation(self):
|
| 1007 |
+
"""Resume the DP engines while request admission remains closed."""
|
| 1008 |
+
if self.node_rank == 0:
|
| 1009 |
+
await self.engine.resume_generation()
|
| 1010 |
+
|
| 1011 |
+
async def open_submission_gate(self):
|
| 1012 |
+
"""Admit requests after the replica has finished resuming its engines."""
|
| 1013 |
+
self._rejecting = False
|
| 1014 |
+
self._submission_paused = False
|
| 1015 |
+
self._resume_event.set()
|
| 1016 |
+
|
| 1017 |
+
async def resume_generation(self):
|
| 1018 |
+
"""Resume generation after abort_all_requests (pause_generation)."""
|
| 1019 |
+
await self.resume_engine_generation()
|
| 1020 |
+
await self.open_submission_gate()
|
| 1021 |
+
|
| 1022 |
+
async def _park_until_admitted(self, request_id: str) -> Optional[TokenOutput]:
|
| 1023 |
+
"""Wait out a closed gate, or fail the request when the gate rejects late arrivals.
|
| 1024 |
+
|
| 1025 |
+
Returns None once the request is admitted, or an aborted TokenOutput that tells the
|
| 1026 |
+
client to re-route. Rejection is checked before waiting, so a rejecting gate never
|
| 1027 |
+
parks anyone and the request never reaches the engine.
|
| 1028 |
+
"""
|
| 1029 |
+
while self._submission_paused:
|
| 1030 |
+
if self._rejecting:
|
| 1031 |
+
logger.debug("rejecting request %s: server left rotation behind a closed gate", request_id)
|
| 1032 |
+
return TokenOutput(
|
| 1033 |
+
token_ids=[],
|
| 1034 |
+
log_probs=None,
|
| 1035 |
+
routed_experts=None,
|
| 1036 |
+
stop_reason="aborted",
|
| 1037 |
+
extra_fields={"global_steps": self.global_steps},
|
| 1038 |
+
)
|
| 1039 |
+
logger.debug("parking request %s until weight sync completes", request_id)
|
| 1040 |
+
await self._resume_event.wait()
|
| 1041 |
+
# No await between the final gate check and the bump: on the actor's single event loop
|
| 1042 |
+
# that keeps "gate closed and _admitting == 0" from being observed mid-admission.
|
| 1043 |
+
self._admitting += 1
|
| 1044 |
+
return None
|
| 1045 |
+
|
| 1046 |
+
async def abort_request(self, request_id: str, reset_prefix_cache: bool = True) -> dict[str, Any]:
|
| 1047 |
+
"""Abort a specific generation request.
|
| 1048 |
+
|
| 1049 |
+
Args:
|
| 1050 |
+
request_id: The ID of the request to abort.
|
| 1051 |
+
|
| 1052 |
+
Returns:
|
| 1053 |
+
dict[str, Any]: Dictionary containing abort result.
|
| 1054 |
+
"""
|
| 1055 |
+
if self.node_rank != 0:
|
| 1056 |
+
return {"aborted": False, "request_id": request_id}
|
| 1057 |
+
|
| 1058 |
+
try:
|
| 1059 |
+
request_states = self.engine.output_processor.request_states
|
| 1060 |
+
req_state = request_states.get(request_id)
|
| 1061 |
+
|
| 1062 |
+
if req_state is None:
|
| 1063 |
+
return {"aborted": False, "error": f"Request {request_id} not found"}
|
| 1064 |
+
|
| 1065 |
+
# Create abort output and put it to the queue
|
| 1066 |
+
from vllm.v1.engine import FinishReason
|
| 1067 |
+
|
| 1068 |
+
request_output = req_state.make_request_output(
|
| 1069 |
+
[], pooling_output=None, finish_reason=FinishReason.ABORT, stop_reason=None
|
| 1070 |
+
)
|
| 1071 |
+
req_state.queue.put(request_output)
|
| 1072 |
+
|
| 1073 |
+
# Abort in output processor and engine core
|
| 1074 |
+
self.engine.output_processor.abort_requests([request_id])
|
| 1075 |
+
await self.engine.engine_core.abort_requests_async([request_id])
|
| 1076 |
+
|
| 1077 |
+
# Try to reset prefix cache to ensure clean state
|
| 1078 |
+
if reset_prefix_cache:
|
| 1079 |
+
await self.clear_kv_cache()
|
| 1080 |
+
logger.info(f"Prefix cache reset after abort request {request_id}")
|
| 1081 |
+
|
| 1082 |
+
logger.info(f"Aborted request: {request_id}")
|
| 1083 |
+
return {"aborted": True, "request_id": request_id}
|
| 1084 |
+
|
| 1085 |
+
except Exception as e:
|
| 1086 |
+
logger.error(f"Error aborting request {request_id}: {e}")
|
| 1087 |
+
return {"aborted": False, "request_id": request_id, "error": str(e)}
|
| 1088 |
+
|
| 1089 |
+
# -----------------------------------------------------------------------
|
| 1090 |
+
# Hook methods for subclass overrides
|
| 1091 |
+
# -----------------------------------------------------------------------
|
| 1092 |
+
|
| 1093 |
+
def _init_config(self, config):
|
| 1094 |
+
"""Initialise config. Override when a specific dataclass_type is needed."""
|
| 1095 |
+
return omega_conf_to_dataclass(config)
|
| 1096 |
+
|
| 1097 |
+
def _init_model_config(self, model_config):
|
| 1098 |
+
"""Initialise model_config. Override when a specific dataclass_type is needed."""
|
| 1099 |
+
return omega_conf_to_dataclass(model_config, dataclass_type=HFModelConfig)
|
| 1100 |
+
|
| 1101 |
+
def _validate_configs(self) -> None:
|
| 1102 |
+
"""Validate config/model_config after initialisation."""
|
| 1103 |
+
max_position_embeddings = get_max_position_embeddings(self.model_config.hf_config)
|
| 1104 |
+
if self.config.max_model_len is None:
|
| 1105 |
+
self.config.max_model_len = max_position_embeddings
|
| 1106 |
+
else:
|
| 1107 |
+
if self.config.max_model_len > max_position_embeddings:
|
| 1108 |
+
raise ValueError(
|
| 1109 |
+
f"max_model_len ({self.config.max_model_len}) should be less than or equal to "
|
| 1110 |
+
f"max_position_embeddings ({max_position_embeddings})"
|
| 1111 |
+
)
|
| 1112 |
+
|
| 1113 |
+
if not self.config.enable_chunked_prefill and self.config.max_num_batched_tokens < self.config.max_model_len:
|
| 1114 |
+
logger.warning(
|
| 1115 |
+
"enable_chunked_prefill=False requires max_num_batched_tokens >= max_model_len "
|
| 1116 |
+
f"({self.config.max_model_len}); raising max_num_batched_tokens from "
|
| 1117 |
+
f"{self.config.max_num_batched_tokens} to {self.config.max_model_len}."
|
| 1118 |
+
)
|
| 1119 |
+
self.config.max_num_batched_tokens = self.config.max_model_len
|
| 1120 |
+
|
| 1121 |
+
def _post_init(self, cuda_visible_devices: str) -> None:
|
| 1122 |
+
"""Called at the end of __init__. Default logs server metadata."""
|
| 1123 |
+
logger.info(
|
| 1124 |
+
f"{self.__class__.__name__}, replica_rank: {self.replica_rank}, node_rank: {self.node_rank}, "
|
| 1125 |
+
f"{get_visible_devices_keyword()}: {cuda_visible_devices}, "
|
| 1126 |
+
f"master_address: {self._master_address}, master_port: {self._master_port}, "
|
| 1127 |
+
f"data_parallel_rpc_port: {self._dp_rpc_port}, data_parallel_master_port: {self._dp_master_port}"
|
| 1128 |
+
)
|
| 1129 |
+
|
| 1130 |
+
def _get_engine_kwargs_key(self) -> str:
|
| 1131 |
+
"""Return the key under config.engine_kwargs for this engine (e.g. 'vllm')."""
|
| 1132 |
+
return "vllm"
|
| 1133 |
+
|
| 1134 |
+
def _preprocess_engine_kwargs(self, engine_kwargs: dict) -> None:
|
| 1135 |
+
"""Mutate engine_kwargs in-place before the CLI args dict is built."""
|
| 1136 |
+
if _VLLM_VERSION < version.parse("0.22.0"):
|
| 1137 |
+
# Work around multimodal processor cache desync across pause/resume.
|
| 1138 |
+
# See: https://github.com/vllm-project/vllm/pull/43001/
|
| 1139 |
+
engine_kwargs.setdefault("mm_processor_cache_gb", 0)
|
| 1140 |
+
|
| 1141 |
+
checkpoint_config = getattr(self.config, "checkpoint_engine", None)
|
| 1142 |
+
if getattr(checkpoint_config, "backend", None) == "delta_sharded":
|
| 1143 |
+
from verl.workers.rollout.vllm_rollout.delta_weight_transfer import (
|
| 1144 |
+
VERL_DELTA_WEIGHT_TRANSFER_BACKEND,
|
| 1145 |
+
is_moe_model,
|
| 1146 |
+
require_vllm_delta_support,
|
| 1147 |
+
)
|
| 1148 |
+
|
| 1149 |
+
require_vllm_delta_support()
|
| 1150 |
+
delta_engine_kwargs = getattr(checkpoint_config, "engine_kwargs", {}).get("delta_sharded", {})
|
| 1151 |
+
if int(delta_engine_kwargs.get("verify_every", 0)) > 0:
|
| 1152 |
+
raise NotImplementedError("delta_sharded with vLLM does not support verify_every > 0")
|
| 1153 |
+
if self.config.data_parallel_size != 1:
|
| 1154 |
+
raise NotImplementedError("delta_sharded with vLLM requires data_parallel_size=1")
|
| 1155 |
+
if self.config.disaggregation.enabled:
|
| 1156 |
+
raise NotImplementedError("delta_sharded with vLLM does not support PD disaggregation")
|
| 1157 |
+
# config.pipeline_model_parallel_size > 1 is already rejected globally;
|
| 1158 |
+
# engine_kwargs is forwarded verbatim to vLLM, so close topology
|
| 1159 |
+
# override paths that would bypass VERL's worker and IPC mapping.
|
| 1160 |
+
if int(engine_kwargs.get("data_parallel_size") or 1) > 1:
|
| 1161 |
+
raise NotImplementedError("delta_sharded with vLLM requires data_parallel_size=1")
|
| 1162 |
+
if int(engine_kwargs.get("pipeline_parallel_size") or 1) > 1:
|
| 1163 |
+
raise NotImplementedError("delta_sharded with vLLM requires pipeline_parallel_size=1")
|
| 1164 |
+
engine_tp_size = engine_kwargs.get("tensor_parallel_size")
|
| 1165 |
+
if engine_tp_size is not None and int(engine_tp_size) != self.config.tensor_model_parallel_size:
|
| 1166 |
+
raise NotImplementedError(
|
| 1167 |
+
"delta_sharded with vLLM requires engine_kwargs tensor_parallel_size to match "
|
| 1168 |
+
"rollout.tensor_model_parallel_size"
|
| 1169 |
+
)
|
| 1170 |
+
|
| 1171 |
+
if is_moe_model(self.model_config.hf_config):
|
| 1172 |
+
moe_backend = engine_kwargs.get("moe_backend")
|
| 1173 |
+
if moe_backend not in {None, "auto", "triton"}:
|
| 1174 |
+
raise NotImplementedError(
|
| 1175 |
+
f"delta_sharded with vLLM MoE requires moe_backend='triton'; got {moe_backend!r}"
|
| 1176 |
+
)
|
| 1177 |
+
engine_kwargs["moe_backend"] = "triton"
|
| 1178 |
+
if engine_kwargs.get("enable_eplb", False):
|
| 1179 |
+
raise NotImplementedError("delta_sharded with vLLM MoE does not support EPLB")
|
| 1180 |
+
|
| 1181 |
+
weight_transfer_config = engine_kwargs.get("weight_transfer_config")
|
| 1182 |
+
if weight_transfer_config is None:
|
| 1183 |
+
weight_transfer_backend = None
|
| 1184 |
+
elif isinstance(weight_transfer_config, Mapping):
|
| 1185 |
+
weight_transfer_backend = weight_transfer_config.get("backend")
|
| 1186 |
+
else:
|
| 1187 |
+
raise TypeError("weight_transfer_config must be a mapping when using delta_sharded")
|
| 1188 |
+
|
| 1189 |
+
if weight_transfer_backend not in {None, VERL_DELTA_WEIGHT_TRANSFER_BACKEND}:
|
| 1190 |
+
raise ValueError(
|
| 1191 |
+
"checkpoint_engine.backend='delta_sharded' requires vLLM "
|
| 1192 |
+
f"weight transfer backend {VERL_DELTA_WEIGHT_TRANSFER_BACKEND!r}, "
|
| 1193 |
+
f"but got {weight_transfer_backend!r}"
|
| 1194 |
+
)
|
| 1195 |
+
engine_kwargs["weight_transfer_config"] = {"backend": VERL_DELTA_WEIGHT_TRANSFER_BACKEND}
|
| 1196 |
+
|
| 1197 |
+
def _get_override_generation_config(self) -> dict:
|
| 1198 |
+
"""Return the override_generation_config dict."""
|
| 1199 |
+
# Override default generation config from hugging face model config,
|
| 1200 |
+
# user can still override them by passing kwargs in each request.
|
| 1201 |
+
return dict(
|
| 1202 |
+
temperature=self.config.temperature,
|
| 1203 |
+
top_k=self.config.top_k,
|
| 1204 |
+
top_p=self.config.top_p,
|
| 1205 |
+
repetition_penalty=1.0,
|
| 1206 |
+
max_new_tokens=self.config.response_length,
|
| 1207 |
+
)
|
| 1208 |
+
|
| 1209 |
+
def _apply_quantization(self) -> tuple[Optional[str], dict]:
|
| 1210 |
+
"""Process quantization config. Returns (quantization_str, hf_overrides)."""
|
| 1211 |
+
quantization = self.config.quantization
|
| 1212 |
+
hf_overrides = {}
|
| 1213 |
+
|
| 1214 |
+
# Handle QAT (Quantization-Aware Training) configuration
|
| 1215 |
+
qat_config_dict = getattr(self.config, "qat", {}) or {}
|
| 1216 |
+
if qat_config_dict.get("enable", False):
|
| 1217 |
+
from verl.utils.qat import QATConfig, load_quantization_config
|
| 1218 |
+
|
| 1219 |
+
qat_config = QATConfig(**qat_config_dict)
|
| 1220 |
+
quantization_config_dict = load_quantization_config(qat_config)
|
| 1221 |
+
quant_method = quantization_config_dict.get("quant_method", None)
|
| 1222 |
+
|
| 1223 |
+
if quant_method == "modelopt":
|
| 1224 |
+
from verl.utils.modelopt import apply_modelopt_nvfp4_patches
|
| 1225 |
+
|
| 1226 |
+
apply_modelopt_nvfp4_patches()
|
| 1227 |
+
quantization = "modelopt"
|
| 1228 |
+
elif quant_method == "compressed-tensors":
|
| 1229 |
+
from verl.utils.qat import apply_qat_patches
|
| 1230 |
+
|
| 1231 |
+
apply_qat_patches()
|
| 1232 |
+
quantization = "compressed-tensors"
|
| 1233 |
+
else:
|
| 1234 |
+
raise ValueError(f"Unsupported quant_method: {quant_method}")
|
| 1235 |
+
|
| 1236 |
+
logger.info(f"QAT quantization config injected (quant_method={quant_method})")
|
| 1237 |
+
hf_overrides["quantization_config"] = quantization_config_dict
|
| 1238 |
+
elif quantization is not None:
|
| 1239 |
+
# Handle other quantization methods (fp8, torchao)
|
| 1240 |
+
_SUPPORTED_QUANTIZATION = ["fp8", "torchao", "ascend"]
|
| 1241 |
+
if quantization not in _SUPPORTED_QUANTIZATION:
|
| 1242 |
+
raise ValueError(f"Currently only support {_SUPPORTED_QUANTIZATION} quantization, got: {quantization}")
|
| 1243 |
+
|
| 1244 |
+
if quantization == "fp8":
|
| 1245 |
+
# Ignore MoE router layers for FP8 quantization
|
| 1246 |
+
all_mlp_gate_layers = []
|
| 1247 |
+
for layer in range(self.model_config.hf_config.num_hidden_layers):
|
| 1248 |
+
all_mlp_gate_layers.append(f"model.layers.{layer}.mlp.gate")
|
| 1249 |
+
|
| 1250 |
+
FP8_BLOCK_QUANT_KWARGS = {
|
| 1251 |
+
"activation_scheme": "dynamic",
|
| 1252 |
+
"fmt": "e4m3",
|
| 1253 |
+
"quant_method": "fp8",
|
| 1254 |
+
"weight_block_size": [128, 128],
|
| 1255 |
+
"ignored_layers": all_mlp_gate_layers,
|
| 1256 |
+
}
|
| 1257 |
+
hf_overrides["quantization_config"] = dict(FP8_BLOCK_QUANT_KWARGS)
|
| 1258 |
+
# Will remove the patch after vllm support on-the-fly quant for rollout natively.
|
| 1259 |
+
apply_vllm_quant_patches()
|
| 1260 |
+
# for subprocesses patching
|
| 1261 |
+
os.environ["VERL_VLLM_FP8_QUANT_ENABLED"] = "1"
|
| 1262 |
+
|
| 1263 |
+
model_quantization_config = getattr(self.model_config.hf_config, "quantization_config", {}) or {}
|
| 1264 |
+
if quantization is None and model_quantization_config.get("quant_method") == "fp8":
|
| 1265 |
+
apply_vllm_quant_patches()
|
| 1266 |
+
os.environ["VERL_VLLM_FP8_QUANT_ENABLED"] = "1"
|
| 1267 |
+
|
| 1268 |
+
if quantization is not None and self.config.quantization_config_file is not None:
|
| 1269 |
+
hf_overrides["quantization_config_file"] = self.config.quantization_config_file
|
| 1270 |
+
|
| 1271 |
+
return quantization, hf_overrides
|
| 1272 |
+
|
| 1273 |
+
def _get_worker_extension_cls(self) -> str:
|
| 1274 |
+
"""Return the fully-qualified colocate worker extension class name."""
|
| 1275 |
+
return "verl.workers.rollout.vllm_rollout.utils.vLLMColocateWorkerExtension"
|
| 1276 |
+
|
| 1277 |
+
def _get_cli_modules(self) -> list:
|
| 1278 |
+
"""Return the list of CLI command modules used for argument parsing."""
|
| 1279 |
+
return [vllm.entrypoints.cli.serve]
|
| 1280 |
+
|
| 1281 |
+
def _get_cli_description(self) -> str:
|
| 1282 |
+
"""Return the description string for the CLI argument parser."""
|
| 1283 |
+
return "vLLM CLI"
|
| 1284 |
+
|
| 1285 |
+
def _get_wake_up_tags(self) -> list[str]:
|
| 1286 |
+
"""Return the tags passed to engine.wake_up(). Default includes kv_cache."""
|
| 1287 |
+
return ["kv_cache", "weights"]
|
| 1288 |
+
|
| 1289 |
+
def _resolve_sleep_level(self) -> int:
|
| 1290 |
+
"""Deepest sleep level whose discarded state a subsequent weight sync can restore.
|
| 1291 |
+
|
| 1292 |
+
MTP drafter-only weights are initialized by vLLM and are not guaranteed
|
| 1293 |
+
to be restored by actor weight sync after level 2 sleep discards them.
|
| 1294 |
+
lora only update adapter weights, so set sleep level to 1.
|
| 1295 |
+
vllm_ascend not support sleep_level now. Enabling EP during training may lead to accuracy issues.
|
| 1296 |
+
"""
|
| 1297 |
+
mtp_config = getattr(self.config, "mtp", None)
|
| 1298 |
+
mtp_rollout_enabled = (
|
| 1299 |
+
mtp_config is not None
|
| 1300 |
+
and getattr(mtp_config, "enable", False)
|
| 1301 |
+
and getattr(mtp_config, "enable_rollout", False)
|
| 1302 |
+
)
|
| 1303 |
+
if mtp_rollout_enabled or self.lora_as_adapter or is_torch_npu_available(check_device=False):
|
| 1304 |
+
return 1
|
| 1305 |
+
return 2
|
| 1306 |
+
|
| 1307 |
+
async def _sleep_hybrid(self):
|
| 1308 |
+
"""HYBRID sleep: adapters and MTP need level=1; full weights need level=2.
|
| 1309 |
+
|
| 1310 |
+
Uses engine.sleep() instead of engine.collective_rpc("sleep") to ensure
|
| 1311 |
+
that sleep is properly propagated to all data-parallel worker processes.
|
| 1312 |
+
collective_rpc only reaches the TP workers within a single DP shard,
|
| 1313 |
+
leaving other DP shards' weights unreleased, which causes OOM during
|
| 1314 |
+
FSDP training backward when DP > 1.
|
| 1315 |
+
"""
|
| 1316 |
+
await self.engine.sleep(level=self._resolve_sleep_level())
|
| 1317 |
+
await self.engine.reset_encoder_cache()
|
| 1318 |
+
|
| 1319 |
+
|
| 1320 |
+
class vLLMReplica(RolloutReplica):
|
| 1321 |
+
def __init__(
|
| 1322 |
+
self,
|
| 1323 |
+
replica_rank: int,
|
| 1324 |
+
config: RolloutConfig,
|
| 1325 |
+
model_config: HFModelConfig,
|
| 1326 |
+
gpus_per_node: int = 8,
|
| 1327 |
+
is_reward_model: bool = False,
|
| 1328 |
+
is_teacher_model: bool = False,
|
| 1329 |
+
name_suffix: str = "",
|
| 1330 |
+
):
|
| 1331 |
+
super().__init__(
|
| 1332 |
+
replica_rank, config, model_config, gpus_per_node, is_reward_model, is_teacher_model, name_suffix
|
| 1333 |
+
)
|
| 1334 |
+
self.server_class = ray.remote(vLLMHttpServer)
|
| 1335 |
+
|
| 1336 |
+
async def launch_servers(self):
|
| 1337 |
+
"""Launch http server in each node."""
|
| 1338 |
+
assert len(self.workers) == self.world_size, (
|
| 1339 |
+
f"worker number {len(self.workers)} not equal to world size {self.world_size}"
|
| 1340 |
+
)
|
| 1341 |
+
|
| 1342 |
+
# get (node_id, CUDA_VISIBLE_DEVICES) of all workers
|
| 1343 |
+
worker_infos = await asyncio.gather(
|
| 1344 |
+
*[
|
| 1345 |
+
worker.__ray_call__.remote(
|
| 1346 |
+
lambda self: (
|
| 1347 |
+
ray.get_runtime_context().get_node_id(),
|
| 1348 |
+
ray.get_runtime_context().get_accelerator_ids()[get_resource_name()][0],
|
| 1349 |
+
)
|
| 1350 |
+
)
|
| 1351 |
+
for worker in self.workers
|
| 1352 |
+
]
|
| 1353 |
+
)
|
| 1354 |
+
worker_cuda_visible_devices = [worker_info[1] for worker_info in worker_infos]
|
| 1355 |
+
worker_node_ids = [worker_info[0] for worker_info in worker_infos]
|
| 1356 |
+
|
| 1357 |
+
# create server actor in each node with node affinity and cuda visible devices
|
| 1358 |
+
nnodes, gpus_per_replica_node = self.nnodes, self.gpus_per_replica_node
|
| 1359 |
+
for node_rank in range(nnodes):
|
| 1360 |
+
workers = self.workers[node_rank * gpus_per_replica_node : (node_rank + 1) * gpus_per_replica_node]
|
| 1361 |
+
node_cuda_visible_devices = ",".join(
|
| 1362 |
+
worker_cuda_visible_devices[node_rank * gpus_per_replica_node : (node_rank + 1) * gpus_per_replica_node]
|
| 1363 |
+
)
|
| 1364 |
+
node_id = worker_node_ids[node_rank * gpus_per_replica_node]
|
| 1365 |
+
prefix = self._get_server_name_prefix()
|
| 1366 |
+
if self.is_reward_model:
|
| 1367 |
+
name = f"{prefix}server_reward_{self.replica_rank}_{node_rank}{self.name_suffix}"
|
| 1368 |
+
elif self.is_teacher_model:
|
| 1369 |
+
name = f"{prefix}server_teacher_{self.replica_rank}_{node_rank}{self.name_suffix}"
|
| 1370 |
+
else:
|
| 1371 |
+
name = f"{prefix}server_{self.replica_rank}_{node_rank}{self.name_suffix}"
|
| 1372 |
+
env_vars = {
|
| 1373 |
+
**{var: "1" for var in get_platform().ray_noset_envvars()},
|
| 1374 |
+
**get_platform().rollout_env_vars(),
|
| 1375 |
+
}
|
| 1376 |
+
|
| 1377 |
+
server = self.server_class.options(
|
| 1378 |
+
scheduling_strategy=ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy(
|
| 1379 |
+
node_id=node_id,
|
| 1380 |
+
soft=False,
|
| 1381 |
+
),
|
| 1382 |
+
runtime_env={"env_vars": env_vars},
|
| 1383 |
+
name=name,
|
| 1384 |
+
max_concurrency=self.max_concurrency,
|
| 1385 |
+
).remote(
|
| 1386 |
+
config=self.config,
|
| 1387 |
+
model_config=self.model_config,
|
| 1388 |
+
rollout_mode=self.rollout_mode,
|
| 1389 |
+
workers=workers,
|
| 1390 |
+
replica_rank=self.replica_rank,
|
| 1391 |
+
node_rank=node_rank,
|
| 1392 |
+
gpus_per_node=gpus_per_replica_node,
|
| 1393 |
+
nnodes=nnodes,
|
| 1394 |
+
cuda_visible_devices=node_cuda_visible_devices,
|
| 1395 |
+
)
|
| 1396 |
+
self.servers.append(server)
|
| 1397 |
+
|
| 1398 |
+
# launch http server in each node
|
| 1399 |
+
master_address, master_port, dp_rpc_port = await self.servers[0].get_master_address.remote()
|
| 1400 |
+
await asyncio.gather(
|
| 1401 |
+
*[
|
| 1402 |
+
server.launch_server.remote(
|
| 1403 |
+
master_address=master_address, master_port=master_port, dp_rpc_port=dp_rpc_port
|
| 1404 |
+
)
|
| 1405 |
+
for server in self.servers
|
| 1406 |
+
]
|
| 1407 |
+
)
|
| 1408 |
+
|
| 1409 |
+
# get http server address from first server
|
| 1410 |
+
server_address, server_port = await self.servers[0].get_server_address.remote()
|
| 1411 |
+
self._server_handle = self.servers[0]
|
| 1412 |
+
self._server_address = (
|
| 1413 |
+
f"[{server_address}]:{server_port}"
|
| 1414 |
+
if is_valid_ipv6_address(server_address)
|
| 1415 |
+
else f"{server_address}:{server_port}"
|
| 1416 |
+
)
|
| 1417 |
+
|
| 1418 |
+
async def sleep(self):
|
| 1419 |
+
"""Sleep each rollout server."""
|
| 1420 |
+
# Drain DP engines for safe sleep.
|
| 1421 |
+
await self.servers[0].wait_for_requests_to_drain.remote()
|
| 1422 |
+
await asyncio.gather(*[server.sleep.remote() for server in self.servers])
|
| 1423 |
+
|
| 1424 |
+
async def abort_all_requests(self, reject_request: bool = False) -> dict[str, Any]:
|
| 1425 |
+
"""Abort all ongoing generation requests across all servers.
|
| 1426 |
+
|
| 1427 |
+
Args:
|
| 1428 |
+
reject_request: Fail requests arriving behind the closed gate instead of
|
| 1429 |
+
parking them. See vLLMHttpServer.abort_all_requests().
|
| 1430 |
+
|
| 1431 |
+
Returns:
|
| 1432 |
+
dict[str, Any]: Combined abort results from all servers.
|
| 1433 |
+
"""
|
| 1434 |
+
results = await asyncio.gather(
|
| 1435 |
+
*[server.abort_all_requests.remote(reject_request=reject_request) for server in self.servers]
|
| 1436 |
+
)
|
| 1437 |
+
|
| 1438 |
+
total_aborted = sum(r.get("aborted_count", 0) for r in results)
|
| 1439 |
+
all_request_ids = []
|
| 1440 |
+
for r in results:
|
| 1441 |
+
all_request_ids.extend(r.get("request_ids", []))
|
| 1442 |
+
|
| 1443 |
+
return {
|
| 1444 |
+
"aborted_count": total_aborted,
|
| 1445 |
+
"request_ids": all_request_ids,
|
| 1446 |
+
"server_results": results,
|
| 1447 |
+
}
|
| 1448 |
+
|
| 1449 |
+
async def resume_generation(self):
|
| 1450 |
+
"""Resume the DP engines before admitting requests on any server."""
|
| 1451 |
+
# KV-cache wake-up may already have resumed scheduling. Keep admission
|
| 1452 |
+
# closed during this resume too: new requests could otherwise start a
|
| 1453 |
+
# DP wave while only some ranks are still in the resume collective.
|
| 1454 |
+
await self.servers[0].resume_engine_generation.remote()
|
| 1455 |
+
await asyncio.gather(*[server.open_submission_gate.remote() for server in self.servers])
|
| 1456 |
+
|
| 1457 |
+
async def abort_request(self, request_id: str) -> dict[str, Any]:
|
| 1458 |
+
"""Abort a specific request. Tries all servers since we don't know which one has it.
|
| 1459 |
+
|
| 1460 |
+
Args:
|
| 1461 |
+
request_id: The ID of the request to abort.
|
| 1462 |
+
|
| 1463 |
+
Returns:
|
| 1464 |
+
dict[str, Any]: Abort result.
|
| 1465 |
+
"""
|
| 1466 |
+
# TODO(petersh6): we should only abort on the server that has the request.
|
| 1467 |
+
results = await asyncio.gather(*[server.abort_request.remote(request_id) for server in self.servers])
|
| 1468 |
+
|
| 1469 |
+
for r in results:
|
| 1470 |
+
if r.get("aborted", False):
|
| 1471 |
+
return r
|
| 1472 |
+
|
| 1473 |
+
return {"aborted": False, "request_id": request_id, "error": "Request not found on any server"}
|
| 1474 |
+
|
| 1475 |
+
async def release_kv_cache(self):
|
| 1476 |
+
# Drain all in-flight requests so that vLLM worker threads go idle
|
| 1477 |
+
# before we touch engine.release_kv_cache()
|
| 1478 |
+
await self.servers[0].wait_for_requests_to_drain.remote()
|
| 1479 |
+
await asyncio.gather(*[server.release_kv_cache.remote() for server in self.servers])
|
| 1480 |
+
|
| 1481 |
+
# -----------------------------------------------------------------------
|
| 1482 |
+
# Hook methods for subclass overrides
|
| 1483 |
+
# -----------------------------------------------------------------------
|
| 1484 |
+
|
| 1485 |
+
def _get_server_name_prefix(self) -> str:
|
| 1486 |
+
"""Return the Ray actor name prefix (e.g. 'vllm_')."""
|
| 1487 |
+
return "vllm_"
|
repro/vendor/verl/workers/rollout/vllm_rollout/vllm_pd_replica.py
ADDED
|
@@ -0,0 +1,305 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 Bytedance Ltd. and/or its affiliates
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""vLLM PD-disaggregated replica: 1 prefill + N decode servers per replica,
|
| 15 |
+
asymmetric TP supported. MVP: prefill_replicas=1, single-node only."""
|
| 16 |
+
|
| 17 |
+
import asyncio
|
| 18 |
+
import logging
|
| 19 |
+
import os
|
| 20 |
+
import uuid
|
| 21 |
+
from dataclasses import replace as _dc_replace
|
| 22 |
+
from typing import Optional
|
| 23 |
+
|
| 24 |
+
import ray
|
| 25 |
+
from ray.actor import ActorHandle
|
| 26 |
+
|
| 27 |
+
from verl.utils.device import get_device_name, get_resource_name, is_torch_npu_available
|
| 28 |
+
from verl.utils.net_utils import get_free_port, is_valid_ipv6_address
|
| 29 |
+
from verl.workers.config import HFModelConfig, RolloutConfig
|
| 30 |
+
from verl.workers.rollout.vllm_rollout.vllm_async_server import vLLMReplica
|
| 31 |
+
|
| 32 |
+
logger = logging.getLogger(__file__)
|
| 33 |
+
logger.setLevel(logging.INFO)
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class vLLMPDReplica(vLLMReplica):
|
| 37 |
+
"""Replica that runs vLLM in prefill-decode disaggregated mode."""
|
| 38 |
+
|
| 39 |
+
def __init__(
|
| 40 |
+
self,
|
| 41 |
+
replica_rank: int,
|
| 42 |
+
config: RolloutConfig,
|
| 43 |
+
model_config: HFModelConfig,
|
| 44 |
+
gpus_per_node: int = 8,
|
| 45 |
+
is_reward_model: bool = False,
|
| 46 |
+
is_teacher_model: bool = False,
|
| 47 |
+
name_suffix: str = "",
|
| 48 |
+
):
|
| 49 |
+
super().__init__(
|
| 50 |
+
replica_rank,
|
| 51 |
+
config,
|
| 52 |
+
model_config,
|
| 53 |
+
gpus_per_node,
|
| 54 |
+
is_reward_model,
|
| 55 |
+
is_teacher_model,
|
| 56 |
+
name_suffix,
|
| 57 |
+
)
|
| 58 |
+
|
| 59 |
+
disagg = self.config.disaggregation
|
| 60 |
+
assert disagg.enabled, "vLLMPDReplica requires rollout.disaggregation.enabled=True"
|
| 61 |
+
|
| 62 |
+
if disagg.transfer_backend not in ("nixl", "mooncake"):
|
| 63 |
+
raise NotImplementedError(
|
| 64 |
+
f"vLLMPDReplica supports transfer_backend in ('nixl', 'mooncake') in this "
|
| 65 |
+
f"revision; got {disagg.transfer_backend!r}. mori/ascend/fake are reserved "
|
| 66 |
+
f"in DisaggregationConfig and will land in follow-ups."
|
| 67 |
+
)
|
| 68 |
+
if disagg.prefill_replicas != 1:
|
| 69 |
+
raise NotImplementedError(f"prefill_replicas=1 only (got {disagg.prefill_replicas})")
|
| 70 |
+
self._n_prefill = disagg.prefill_replicas
|
| 71 |
+
self._n_decode = disagg.decode_replicas
|
| 72 |
+
|
| 73 |
+
self._prefill_tp = self.config.tensor_model_parallel_size
|
| 74 |
+
# Inline decode_tp default: OmegaConf/Ray serialization drops dataclass methods.
|
| 75 |
+
self._decode_tp = (
|
| 76 |
+
disagg.decode_tensor_model_parallel_size
|
| 77 |
+
if disagg.decode_tensor_model_parallel_size is not None
|
| 78 |
+
else self._prefill_tp
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
pd_world_size = self._prefill_tp + self._n_decode * self._decode_tp
|
| 82 |
+
if pd_world_size > gpus_per_node:
|
| 83 |
+
raise NotImplementedError(
|
| 84 |
+
f"PD replica needs {pd_world_size} GPUs but gpus_per_node={gpus_per_node}; "
|
| 85 |
+
f"single-node only in this revision (use more replicas to span nodes once "
|
| 86 |
+
f"multi-node lands)"
|
| 87 |
+
)
|
| 88 |
+
if self.config.data_parallel_size != 1:
|
| 89 |
+
raise NotImplementedError(f"data_parallel_size=1 only (got {self.config.data_parallel_size})")
|
| 90 |
+
if self.config.pipeline_model_parallel_size != 1:
|
| 91 |
+
raise NotImplementedError(
|
| 92 |
+
f"pipeline_model_parallel_size=1 only "
|
| 93 |
+
f"(got {self.config.pipeline_model_parallel_size}); PD path does not model PP yet"
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
self.world_size = pd_world_size
|
| 97 |
+
self.gpus_per_replica_node = min(self.gpus_per_node, self.world_size)
|
| 98 |
+
assert self.world_size % self.gpus_per_replica_node == 0
|
| 99 |
+
self.nnodes = self.world_size // self.gpus_per_replica_node
|
| 100 |
+
|
| 101 |
+
self._prefill_servers: list[ActorHandle] = []
|
| 102 |
+
self._decode_servers: list[ActorHandle] = []
|
| 103 |
+
|
| 104 |
+
async def launch_servers(self):
|
| 105 |
+
assert len(self.workers) == self.world_size, (
|
| 106 |
+
f"worker count {len(self.workers)} != PD world size {self.world_size}"
|
| 107 |
+
)
|
| 108 |
+
assert not is_torch_npu_available(check_device=False), "vLLM PD on NPU not validated"
|
| 109 |
+
|
| 110 |
+
worker_infos = await asyncio.gather(
|
| 111 |
+
*[
|
| 112 |
+
worker.__ray_call__.remote(
|
| 113 |
+
lambda self: (
|
| 114 |
+
ray.get_runtime_context().get_node_id(),
|
| 115 |
+
ray.get_runtime_context().get_accelerator_ids()[get_resource_name()][0],
|
| 116 |
+
ray.util.get_node_ip_address().strip("[]"),
|
| 117 |
+
)
|
| 118 |
+
)
|
| 119 |
+
for worker in self.workers
|
| 120 |
+
]
|
| 121 |
+
)
|
| 122 |
+
|
| 123 |
+
# Bind the side-channel on the prefill worker's node.
|
| 124 |
+
prefill_host_ip = worker_infos[0][2]
|
| 125 |
+
prefill_engine_id = uuid.uuid4().hex
|
| 126 |
+
|
| 127 |
+
prefill_end = self._prefill_tp
|
| 128 |
+
prefill_workers = self.workers[0:prefill_end]
|
| 129 |
+
prefill_node_id = worker_infos[0][0]
|
| 130 |
+
prefill_devs = self._collect_cuda_devices(worker_infos[0:prefill_end])
|
| 131 |
+
|
| 132 |
+
# Keep side-channel sockets reserved until all actors bind.
|
| 133 |
+
reserved_socks = []
|
| 134 |
+
prefill_side_channel_port, prefill_sock = get_free_port(prefill_host_ip, with_alive_sock=True)
|
| 135 |
+
reserved_socks.append(prefill_sock)
|
| 136 |
+
try:
|
| 137 |
+
prefill_kv_cfg = self._build_kv_transfer_config(
|
| 138 |
+
role="prefill",
|
| 139 |
+
engine_id=prefill_engine_id,
|
| 140 |
+
transfer_backend=self.config.disaggregation.transfer_backend,
|
| 141 |
+
mooncake_protocol=self.config.disaggregation.mooncake_protocol,
|
| 142 |
+
)
|
| 143 |
+
self._prefill_servers = [
|
| 144 |
+
self._spawn_pd_server(
|
| 145 |
+
role="prefill",
|
| 146 |
+
workers=prefill_workers,
|
| 147 |
+
node_id=prefill_node_id,
|
| 148 |
+
cuda_visible_devices=prefill_devs,
|
| 149 |
+
tp=self._prefill_tp,
|
| 150 |
+
kv_transfer_config=prefill_kv_cfg,
|
| 151 |
+
side_channel_host=prefill_host_ip,
|
| 152 |
+
side_channel_port=prefill_side_channel_port,
|
| 153 |
+
mooncake_bootstrap_port=prefill_side_channel_port,
|
| 154 |
+
actor_name=f"vllm_server_{self.replica_rank}_0{self.name_suffix}",
|
| 155 |
+
zmq_base_trainer_rank=0,
|
| 156 |
+
)
|
| 157 |
+
]
|
| 158 |
+
|
| 159 |
+
for i in range(self._n_decode):
|
| 160 |
+
start = self._prefill_tp + i * self._decode_tp
|
| 161 |
+
end = start + self._decode_tp
|
| 162 |
+
workers_i = self.workers[start:end]
|
| 163 |
+
node_id_i = worker_infos[start][0]
|
| 164 |
+
devs_i = self._collect_cuda_devices(worker_infos[start:end])
|
| 165 |
+
|
| 166 |
+
decode_side_channel_port, decode_sock = get_free_port(prefill_host_ip, with_alive_sock=True)
|
| 167 |
+
reserved_socks.append(decode_sock)
|
| 168 |
+
decode_kv_cfg = self._build_kv_transfer_config(
|
| 169 |
+
role="decode",
|
| 170 |
+
engine_id=uuid.uuid4().hex,
|
| 171 |
+
transfer_backend=self.config.disaggregation.transfer_backend,
|
| 172 |
+
mooncake_protocol=self.config.disaggregation.mooncake_protocol,
|
| 173 |
+
)
|
| 174 |
+
self._decode_servers.append(
|
| 175 |
+
self._spawn_pd_server(
|
| 176 |
+
role="decode",
|
| 177 |
+
workers=workers_i,
|
| 178 |
+
node_id=node_id_i,
|
| 179 |
+
cuda_visible_devices=devs_i,
|
| 180 |
+
tp=self._decode_tp,
|
| 181 |
+
kv_transfer_config=decode_kv_cfg,
|
| 182 |
+
side_channel_host=prefill_host_ip,
|
| 183 |
+
side_channel_port=decode_side_channel_port,
|
| 184 |
+
mooncake_bootstrap_port=prefill_side_channel_port,
|
| 185 |
+
actor_name=f"vllm_server_decode_{self.replica_rank}_{i}{self.name_suffix}",
|
| 186 |
+
zmq_base_trainer_rank=start,
|
| 187 |
+
)
|
| 188 |
+
)
|
| 189 |
+
|
| 190 |
+
await asyncio.gather(
|
| 191 |
+
*[
|
| 192 |
+
server.launch_server.remote(master_address=None, master_port=None, dp_rpc_port=None)
|
| 193 |
+
for server in self._prefill_servers + self._decode_servers
|
| 194 |
+
]
|
| 195 |
+
)
|
| 196 |
+
finally:
|
| 197 |
+
for sock in reserved_socks:
|
| 198 |
+
sock.close()
|
| 199 |
+
|
| 200 |
+
await self._prefill_servers[0].set_pd_peer.remote(
|
| 201 |
+
self._decode_servers,
|
| 202 |
+
prefill_side_channel_port,
|
| 203 |
+
prefill_engine_id,
|
| 204 |
+
)
|
| 205 |
+
|
| 206 |
+
self.servers = list(self._prefill_servers) + list(self._decode_servers)
|
| 207 |
+
prefill_address, prefill_port = await self._prefill_servers[0].get_server_address.remote()
|
| 208 |
+
self._server_handle = self._prefill_servers[0]
|
| 209 |
+
self._server_address = (
|
| 210 |
+
f"[{prefill_address}]:{prefill_port}"
|
| 211 |
+
if is_valid_ipv6_address(prefill_address)
|
| 212 |
+
else f"{prefill_address}:{prefill_port}"
|
| 213 |
+
)
|
| 214 |
+
|
| 215 |
+
logger.info(
|
| 216 |
+
"vLLMPDReplica rank=%s launched: prefill=%s (engine_id=%s, side_channel=%s:%d), decodes=%d",
|
| 217 |
+
self.replica_rank,
|
| 218 |
+
self._server_address,
|
| 219 |
+
prefill_engine_id,
|
| 220 |
+
prefill_host_ip,
|
| 221 |
+
prefill_side_channel_port,
|
| 222 |
+
len(self._decode_servers),
|
| 223 |
+
)
|
| 224 |
+
|
| 225 |
+
@staticmethod
|
| 226 |
+
def _collect_cuda_devices(worker_infos) -> str:
|
| 227 |
+
return ",".join(worker_info[1] for worker_info in worker_infos)
|
| 228 |
+
|
| 229 |
+
@staticmethod
|
| 230 |
+
def _build_kv_transfer_config(
|
| 231 |
+
role: str,
|
| 232 |
+
engine_id: str,
|
| 233 |
+
transfer_backend: str,
|
| 234 |
+
mooncake_protocol: Optional[str] = None,
|
| 235 |
+
) -> dict:
|
| 236 |
+
"""Assemble vLLM's ``--kv-transfer-config`` payload."""
|
| 237 |
+
role_to_kv_role = {
|
| 238 |
+
"prefill": "kv_producer",
|
| 239 |
+
"decode": "kv_consumer",
|
| 240 |
+
}
|
| 241 |
+
connector = {
|
| 242 |
+
"nixl": "NixlConnector",
|
| 243 |
+
"mooncake": "MooncakeConnector",
|
| 244 |
+
}[transfer_backend]
|
| 245 |
+
cfg: dict = {
|
| 246 |
+
"kv_connector": connector,
|
| 247 |
+
"kv_role": role_to_kv_role[role],
|
| 248 |
+
"engine_id": engine_id,
|
| 249 |
+
"kv_buffer_device": get_device_name(),
|
| 250 |
+
}
|
| 251 |
+
if transfer_backend == "mooncake" and mooncake_protocol:
|
| 252 |
+
cfg["kv_connector_extra_config"] = {"mooncake_protocol": mooncake_protocol}
|
| 253 |
+
return cfg
|
| 254 |
+
|
| 255 |
+
def _spawn_pd_server(
|
| 256 |
+
self,
|
| 257 |
+
role: str,
|
| 258 |
+
workers: list[ActorHandle],
|
| 259 |
+
node_id: str,
|
| 260 |
+
cuda_visible_devices: str,
|
| 261 |
+
tp: int,
|
| 262 |
+
kv_transfer_config: dict,
|
| 263 |
+
side_channel_host: str,
|
| 264 |
+
side_channel_port: int,
|
| 265 |
+
mooncake_bootstrap_port: int,
|
| 266 |
+
actor_name: str,
|
| 267 |
+
zmq_base_trainer_rank: int = 0,
|
| 268 |
+
) -> ActorHandle:
|
| 269 |
+
"""Construct one PD ``vLLMHttpServer`` actor."""
|
| 270 |
+
per_role_config = _dc_replace(self.config, tensor_model_parallel_size=tp)
|
| 271 |
+
|
| 272 |
+
env_vars = {
|
| 273 |
+
"RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES": "1",
|
| 274 |
+
"RAY_EXPERIMENTAL_NOSET_ASCEND_RT_VISIBLE_DEVICES": "1",
|
| 275 |
+
"NCCL_CUMEM_ENABLE": "0",
|
| 276 |
+
"VLLM_NIXL_SIDE_CHANNEL_HOST": side_channel_host,
|
| 277 |
+
"VLLM_NIXL_SIDE_CHANNEL_PORT": str(side_channel_port),
|
| 278 |
+
"VLLM_MOONCAKE_BOOTSTRAP_PORT": str(mooncake_bootstrap_port),
|
| 279 |
+
# Avoid Mooncake TCP port exhaustion under validation concurrency.
|
| 280 |
+
"MC_TCP_ENABLE_CONNECTION_POOL": os.environ.get("MC_TCP_ENABLE_CONNECTION_POOL", "1"),
|
| 281 |
+
"VERL_ZMQ_BASE_TRAINER_RANK": str(zmq_base_trainer_rank),
|
| 282 |
+
"VERL_RAY_JOB_ID": ray.get_runtime_context().get_job_id(),
|
| 283 |
+
}
|
| 284 |
+
|
| 285 |
+
return self.server_class.options(
|
| 286 |
+
scheduling_strategy=ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy(
|
| 287 |
+
node_id=node_id,
|
| 288 |
+
soft=False,
|
| 289 |
+
),
|
| 290 |
+
runtime_env={"env_vars": env_vars},
|
| 291 |
+
name=actor_name,
|
| 292 |
+
max_concurrency=self.max_concurrency,
|
| 293 |
+
).remote(
|
| 294 |
+
config=per_role_config,
|
| 295 |
+
model_config=self.model_config,
|
| 296 |
+
rollout_mode=self.rollout_mode,
|
| 297 |
+
workers=workers,
|
| 298 |
+
replica_rank=self.replica_rank,
|
| 299 |
+
node_rank=0,
|
| 300 |
+
gpus_per_node=self.gpus_per_replica_node,
|
| 301 |
+
nnodes=1,
|
| 302 |
+
cuda_visible_devices=cuda_visible_devices,
|
| 303 |
+
disaggregation_role=role,
|
| 304 |
+
disaggregation_kv_transfer_config=kv_transfer_config,
|
| 305 |
+
)
|
repro/vendor/verl/workers/rollout/vllm_rollout/vllm_rollout.py
ADDED
|
@@ -0,0 +1,341 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2024 Bytedance Ltd. and/or its affiliates
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""
|
| 15 |
+
The vllm_rollout that can be applied in different backend
|
| 16 |
+
When working with FSDP:
|
| 17 |
+
- Use DTensor weight loader (recommended) or HF weight loader
|
| 18 |
+
- Utilize state_dict from the FSDP to synchronize the weights among tp ranks in vLLM
|
| 19 |
+
When working with Megatron:
|
| 20 |
+
- Use Megatron weight loader
|
| 21 |
+
- During training, only the current pp stage holds the parameters
|
| 22 |
+
- Before inference, broadcast the parameters of the current pp rank
|
| 23 |
+
to all other pp ranks (all pp ranks holds all the parameters)
|
| 24 |
+
- Bind the parameters to the inference engine
|
| 25 |
+
- Do inference in tp. pp is treated as additional dp
|
| 26 |
+
- After inference, all the parameters that doesn't belong to this pp rank is freed.
|
| 27 |
+
"""
|
| 28 |
+
|
| 29 |
+
import logging
|
| 30 |
+
import os
|
| 31 |
+
import time
|
| 32 |
+
from typing import Any, Generator, Optional
|
| 33 |
+
|
| 34 |
+
import ray
|
| 35 |
+
import torch
|
| 36 |
+
from torch.distributed.device_mesh import DeviceMesh
|
| 37 |
+
|
| 38 |
+
from verl import DataProto
|
| 39 |
+
from verl.third_party.vllm import VLLM_SLEEP_LEVEL
|
| 40 |
+
from verl.utils.device import is_support_ipc
|
| 41 |
+
from verl.workers.config import HFModelConfig, RolloutConfig
|
| 42 |
+
from verl.workers.rollout.base import BaseRollout
|
| 43 |
+
from verl.workers.rollout.vllm_rollout.bucketed_weight_transfer import BucketedWeightSender
|
| 44 |
+
|
| 45 |
+
logger = logging.getLogger(__file__)
|
| 46 |
+
logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "INFO"))
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
class ServerAdapter(BaseRollout):
|
| 50 |
+
"""
|
| 51 |
+
vLLM server adapter used in native async mode, serve as a client to request vLLM server
|
| 52 |
+
to resume/release/update weights and kv_cache.
|
| 53 |
+
"""
|
| 54 |
+
|
| 55 |
+
def __init__(
|
| 56 |
+
self,
|
| 57 |
+
config: RolloutConfig,
|
| 58 |
+
model_config: HFModelConfig,
|
| 59 |
+
device_mesh: DeviceMesh,
|
| 60 |
+
replica_rank: int = -1,
|
| 61 |
+
):
|
| 62 |
+
super().__init__(config, model_config, device_mesh)
|
| 63 |
+
self.server_handle: ray.actor.ActorHandle = None
|
| 64 |
+
|
| 65 |
+
rank = int(os.environ["RANK"])
|
| 66 |
+
local_world_size = int(os.environ["RAY_LOCAL_WORLD_SIZE"])
|
| 67 |
+
# PD asymmetric layout inflates per-replica footprint; must match
|
| 68 |
+
# llm_server.py:_initialize_llm_servers or trainer-to-replica mapping breaks.
|
| 69 |
+
prefill_tp = self.config.tensor_model_parallel_size
|
| 70 |
+
disagg = getattr(self.config, "disaggregation", None)
|
| 71 |
+
if disagg is not None and getattr(disagg, "enabled", False):
|
| 72 |
+
decode_tp = (
|
| 73 |
+
disagg.decode_tensor_model_parallel_size
|
| 74 |
+
if disagg.decode_tensor_model_parallel_size is not None
|
| 75 |
+
else prefill_tp
|
| 76 |
+
)
|
| 77 |
+
per_replica = prefill_tp * disagg.prefill_replicas + decode_tp * disagg.decode_replicas
|
| 78 |
+
else:
|
| 79 |
+
decode_tp = prefill_tp
|
| 80 |
+
per_replica = prefill_tp
|
| 81 |
+
rollout_world_size = per_replica * self.config.data_parallel_size * self.config.pipeline_model_parallel_size
|
| 82 |
+
if replica_rank == -1:
|
| 83 |
+
self.replica_rank = rank // rollout_world_size
|
| 84 |
+
else:
|
| 85 |
+
self.replica_rank = replica_rank
|
| 86 |
+
self.rollout_rank = rank % rollout_world_size
|
| 87 |
+
self.node_rank = self.rollout_rank // local_world_size
|
| 88 |
+
|
| 89 |
+
# Map each trainer rank to its co-located vLLM server so weight-update
|
| 90 |
+
# IPC handles stay on the GPU where they were created. Offset math
|
| 91 |
+
# assumes prefill_replicas == 1 (enforced by vLLMPDReplica); if that
|
| 92 |
+
# ever lifts, update both this block and vLLMPDReplica.launch_servers.
|
| 93 |
+
self._pd_role: Optional[str] = None
|
| 94 |
+
self._pd_server_index: Optional[int] = None
|
| 95 |
+
self._pd_tp_local_rank: Optional[int] = None
|
| 96 |
+
if disagg is not None and getattr(disagg, "enabled", False):
|
| 97 |
+
footprint = prefill_tp + disagg.decode_replicas * decode_tp
|
| 98 |
+
local = self.rollout_rank % footprint
|
| 99 |
+
if local < prefill_tp:
|
| 100 |
+
self._pd_role = "prefill"
|
| 101 |
+
self._pd_server_index = 0
|
| 102 |
+
self._pd_tp_local_rank = local
|
| 103 |
+
else:
|
| 104 |
+
off = local - prefill_tp
|
| 105 |
+
self._pd_role = "decode"
|
| 106 |
+
self._pd_server_index = off // decode_tp
|
| 107 |
+
self._pd_tp_local_rank = off % decode_tp
|
| 108 |
+
# Each role's TP-rank-0 owns the adapter (vs colocated where every
|
| 109 |
+
# rollout_rank=0 owns it). One log line per PD rank at startup so
|
| 110 |
+
# a deadlock report can be traced back to the role mapping.
|
| 111 |
+
self._has_server = self._pd_tp_local_rank == 0
|
| 112 |
+
logger.info(
|
| 113 |
+
"vllm PD ServerAdapter: rank=%d replica=%d rollout=%d role=%s server_idx=%s tp_local=%s has_server=%s",
|
| 114 |
+
rank,
|
| 115 |
+
self.replica_rank,
|
| 116 |
+
self.rollout_rank,
|
| 117 |
+
self._pd_role,
|
| 118 |
+
self._pd_server_index,
|
| 119 |
+
self._pd_tp_local_rank,
|
| 120 |
+
self._has_server,
|
| 121 |
+
)
|
| 122 |
+
else:
|
| 123 |
+
self._has_server = self.rollout_rank == 0
|
| 124 |
+
|
| 125 |
+
if config.layered_summon:
|
| 126 |
+
logger.warning("Setting the sleep level to 1 may cause a memory overflow.")
|
| 127 |
+
self.sleep_level = 1
|
| 128 |
+
else:
|
| 129 |
+
self.sleep_level = VLLM_SLEEP_LEVEL
|
| 130 |
+
|
| 131 |
+
# Use replica_rank + node-local rank to form ZMQ handle instead of GPU UUID,
|
| 132 |
+
# because CheckpointEngineWorker and vLLM worker may see different GPU UUIDs
|
| 133 |
+
# when CUDA_VISIBLE_DEVICES differs between processes (common on ROCm/AMD).
|
| 134 |
+
# Must use node-local rank (not rollout_rank) so it matches vLLM worker's
|
| 135 |
+
# local_rank on every node. Include replica_rank to avoid collisions when
|
| 136 |
+
# multiple replicas share a node, and the Ray job id so two independent
|
| 137 |
+
# verl jobs on the same host (or a new run after a crashed one with a
|
| 138 |
+
# stale socket file) cannot collide on the shared /tmp namespace.
|
| 139 |
+
local_rank = self.rollout_rank % local_world_size
|
| 140 |
+
job_id = ray.get_runtime_context().get_job_id()
|
| 141 |
+
self.zmq_handle = f"ipc:///tmp/rl-colocate-zmq-{job_id}-replica-{self.replica_rank}-rank-{local_rank}.sock"
|
| 142 |
+
|
| 143 |
+
self.use_shm = not is_support_ipc()
|
| 144 |
+
self._delta_weight_transfer_engine_initialized = False
|
| 145 |
+
if self.use_shm:
|
| 146 |
+
logger.warning(
|
| 147 |
+
"IPC is not supported on your devices. Falling back to shared memory for weight transfer, "
|
| 148 |
+
"which may cause performance degradation. If you are using Ascend NPUs, please ensure that "
|
| 149 |
+
"your software and CANN toolkit versions meet the requirements for IPC support. (Ascend HDK version "
|
| 150 |
+
">= 25.3.rc1 and CANN toolkit version >= 8.3.RC1)"
|
| 151 |
+
)
|
| 152 |
+
|
| 153 |
+
def _ensure_server_handle(self) -> bool:
|
| 154 |
+
"""Lazy-init server handle. Returns False if this rank should not proceed."""
|
| 155 |
+
if not self._has_server:
|
| 156 |
+
return False
|
| 157 |
+
# Lazy init http server adapter because http server is launched after hybrid engine.
|
| 158 |
+
if self.server_handle is None:
|
| 159 |
+
prefix = self._get_server_name_prefix()
|
| 160 |
+
if self._pd_role == "prefill":
|
| 161 |
+
actor_name = f"{prefix}server_{self.replica_rank}_0"
|
| 162 |
+
elif self._pd_role == "decode":
|
| 163 |
+
actor_name = f"{prefix}server_decode_{self.replica_rank}_{self._pd_server_index}"
|
| 164 |
+
else:
|
| 165 |
+
actor_name = f"{prefix}server_{self.replica_rank}_{self.node_rank}"
|
| 166 |
+
self.server_handle = ray.get_actor(actor_name)
|
| 167 |
+
return True
|
| 168 |
+
|
| 169 |
+
async def _execute_method(
|
| 170 |
+
self,
|
| 171 |
+
method: str,
|
| 172 |
+
non_block: bool = False,
|
| 173 |
+
timeout: Optional[float] = None,
|
| 174 |
+
args: tuple = (),
|
| 175 |
+
kwargs: Optional[dict] = None,
|
| 176 |
+
) -> Any:
|
| 177 |
+
"""Execute method on inference engine via ray.
|
| 178 |
+
|
| 179 |
+
Args:
|
| 180 |
+
method: The method name to execute on the server.
|
| 181 |
+
non_block: If True, execute the method asynchronously and return immediately.
|
| 182 |
+
timeout: Timeout for the collective_rpc call.
|
| 183 |
+
args: Positional arguments for the method.
|
| 184 |
+
kwargs: Keyword arguments for the method.
|
| 185 |
+
|
| 186 |
+
Returns:
|
| 187 |
+
The result of the method execution, or None if non_block=True.
|
| 188 |
+
"""
|
| 189 |
+
if not self._ensure_server_handle():
|
| 190 |
+
return None
|
| 191 |
+
|
| 192 |
+
future = self.server_handle.collective_rpc.remote(method, timeout=timeout, args=args, kwargs=kwargs)
|
| 193 |
+
return future if non_block else await future
|
| 194 |
+
|
| 195 |
+
async def resume(self, tags: list[str]):
|
| 196 |
+
"""Resume rollout weights or kv cache in GPU memory.
|
| 197 |
+
|
| 198 |
+
Args:
|
| 199 |
+
tags: weights or kv_cache.
|
| 200 |
+
"""
|
| 201 |
+
if self.config.free_cache_engine and self._ensure_server_handle():
|
| 202 |
+
await self.server_handle.wake_up.remote(tags=tags)
|
| 203 |
+
|
| 204 |
+
async def release(self):
|
| 205 |
+
"""Release weights and kv cache in GPU memory."""
|
| 206 |
+
if self.config.free_cache_engine and self._ensure_server_handle():
|
| 207 |
+
await self.server_handle.sleep.remote()
|
| 208 |
+
|
| 209 |
+
@torch.no_grad()
|
| 210 |
+
async def update_weights(
|
| 211 |
+
self,
|
| 212 |
+
weights: Generator[tuple[str, torch.Tensor], None, None],
|
| 213 |
+
global_steps: int = None,
|
| 214 |
+
wire_format: str = "named_tensors",
|
| 215 |
+
**kwargs,
|
| 216 |
+
):
|
| 217 |
+
"""Update rollout weights from full tensors or DeltaFlush payloads.
|
| 218 |
+
|
| 219 |
+
Full-tensor updates may fall back to shared memory; delta updates
|
| 220 |
+
require colocated CUDA IPC.
|
| 221 |
+
"""
|
| 222 |
+
if wire_format == "delta_flush":
|
| 223 |
+
return await self._update_delta_weights(weights, global_steps=global_steps)
|
| 224 |
+
assert wire_format == "named_tensors", (
|
| 225 |
+
f"vLLM rollout supports wire_format='named_tensors' or 'delta_flush'; got {wire_format!r}"
|
| 226 |
+
)
|
| 227 |
+
start_time = time.time()
|
| 228 |
+
|
| 229 |
+
future = await self._execute_method(
|
| 230 |
+
"update_weights_from_ipc",
|
| 231 |
+
non_block=True,
|
| 232 |
+
kwargs={**kwargs, "use_shm": self.use_shm},
|
| 233 |
+
)
|
| 234 |
+
|
| 235 |
+
bucket_size_mb = self.config.checkpoint_engine.update_weights_bucket_megabytes
|
| 236 |
+
sender = BucketedWeightSender(
|
| 237 |
+
zmq_handle=self.zmq_handle,
|
| 238 |
+
bucket_size_mb=bucket_size_mb,
|
| 239 |
+
use_shm=self.use_shm,
|
| 240 |
+
)
|
| 241 |
+
await sender.async_send_weights(weights)
|
| 242 |
+
|
| 243 |
+
if future is not None:
|
| 244 |
+
await future
|
| 245 |
+
|
| 246 |
+
# reset caches after updating weights
|
| 247 |
+
if self._has_server:
|
| 248 |
+
await self.server_handle.clear_kv_cache.remote()
|
| 249 |
+
if global_steps is not None:
|
| 250 |
+
await self.server_handle.set_global_steps.remote(global_steps)
|
| 251 |
+
|
| 252 |
+
if self.replica_rank == 0 and self.rollout_rank == 0:
|
| 253 |
+
logger.info(f"update_weights done, time cost: {time.time() - start_time:.2f}s")
|
| 254 |
+
|
| 255 |
+
async def _update_delta_weights(
|
| 256 |
+
self,
|
| 257 |
+
weights,
|
| 258 |
+
*,
|
| 259 |
+
global_steps: int | None,
|
| 260 |
+
) -> None:
|
| 261 |
+
"""Send one delta weight update as a stream of DeltaFlush payloads."""
|
| 262 |
+
|
| 263 |
+
from verl.workers.rollout.utils import ensure_async_iterator
|
| 264 |
+
|
| 265 |
+
if self.use_shm:
|
| 266 |
+
raise NotImplementedError("delta_sharded with vLLM requires colocated CUDA IPC")
|
| 267 |
+
|
| 268 |
+
flushes = ensure_async_iterator(weights)
|
| 269 |
+
try:
|
| 270 |
+
first_item = await anext(flushes)
|
| 271 |
+
except StopAsyncIteration:
|
| 272 |
+
# A steady sync with no changed BF16 values sends only a terminal
|
| 273 |
+
# marker: weights are unchanged, so advance the step tag without
|
| 274 |
+
# touching the transfer engine or the KV cache.
|
| 275 |
+
if global_steps is not None and self._ensure_server_handle():
|
| 276 |
+
await self.server_handle.set_global_steps.remote(global_steps)
|
| 277 |
+
return
|
| 278 |
+
|
| 279 |
+
first_named_tensors, saw_last = first_item
|
| 280 |
+
if not self._delta_weight_transfer_engine_initialized:
|
| 281 |
+
await self._execute_method(
|
| 282 |
+
"init_weight_transfer_engine",
|
| 283 |
+
kwargs={"init_info": {}},
|
| 284 |
+
)
|
| 285 |
+
self._delta_weight_transfer_engine_initialized = True
|
| 286 |
+
|
| 287 |
+
await self._execute_method("start_weight_update")
|
| 288 |
+
|
| 289 |
+
async def send_flush(flush_tensors: list[tuple[str, torch.Tensor]]) -> None:
|
| 290 |
+
receiver_future = await self._execute_method(
|
| 291 |
+
"update_verl_delta_weights",
|
| 292 |
+
non_block=True,
|
| 293 |
+
kwargs={"update_info": {}},
|
| 294 |
+
)
|
| 295 |
+
sender = BucketedWeightSender(
|
| 296 |
+
zmq_handle=self.zmq_handle,
|
| 297 |
+
bucket_size_mb=self.config.checkpoint_engine.update_weights_bucket_megabytes,
|
| 298 |
+
use_shm=False,
|
| 299 |
+
)
|
| 300 |
+
await sender.async_send_weights(iter(flush_tensors))
|
| 301 |
+
if receiver_future is not None:
|
| 302 |
+
await receiver_future
|
| 303 |
+
|
| 304 |
+
await send_flush(list(first_named_tensors))
|
| 305 |
+
async for named_tensors, is_last in flushes:
|
| 306 |
+
if saw_last:
|
| 307 |
+
raise ValueError("DeltaFlush stream yielded data after is_last=True")
|
| 308 |
+
saw_last = is_last
|
| 309 |
+
await send_flush(list(named_tensors))
|
| 310 |
+
|
| 311 |
+
if not saw_last:
|
| 312 |
+
raise ValueError("DeltaFlush stream ended without is_last=True")
|
| 313 |
+
|
| 314 |
+
await self._execute_method("finish_weight_update")
|
| 315 |
+
|
| 316 |
+
if self._has_server:
|
| 317 |
+
await self.server_handle.clear_kv_cache.remote()
|
| 318 |
+
if global_steps is not None:
|
| 319 |
+
await self.server_handle.set_global_steps.remote(global_steps)
|
| 320 |
+
|
| 321 |
+
def _get_server_name_prefix(self) -> str:
|
| 322 |
+
"""Return the Ray actor name prefix matching the rollout type (e.g. 'vllm_')."""
|
| 323 |
+
return f"{self.config.get('name', 'vllm')}_"
|
| 324 |
+
|
| 325 |
+
def generate_sequences(self, prompts: DataProto) -> DataProto:
|
| 326 |
+
"""Batch generate sequences in sync mode.
|
| 327 |
+
|
| 328 |
+
Note: ServerAdapter uses async server mode and does not support synchronous
|
| 329 |
+
generation. Since SPMD mode was retired (PR #4411), the generation workflow
|
| 330 |
+
should use the async server interface instead.
|
| 331 |
+
|
| 332 |
+
Raises:
|
| 333 |
+
NotImplementedError: Always raised as sync generation is not supported.
|
| 334 |
+
"""
|
| 335 |
+
raise NotImplementedError(
|
| 336 |
+
"ServerAdapter does not support synchronous generate_sequences(). "
|
| 337 |
+
"The vLLM SPMD mode was retired in PR #4411. For batch generation, "
|
| 338 |
+
"please use the async server interface via vLLMReplica and LLMServerClient, "
|
| 339 |
+
"or use HFRollout for synchronous generation. "
|
| 340 |
+
"See https://github.com/verl-project/verl/issues/4682 for more details."
|
| 341 |
+
)
|
repro/vendor/verl/workers/rollout/vllm_rollout/weight_update_utils.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025 Bytedance Ltd. and/or its affiliates
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
|
| 15 |
+
import torch
|
| 16 |
+
|
| 17 |
+
WeightUpdate = tuple[str, torch.Tensor]
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def split_buffer_updates(
|
| 21 |
+
model: torch.nn.Module, weights: list[WeightUpdate]
|
| 22 |
+
) -> tuple[list[WeightUpdate], list[WeightUpdate], dict[str, torch.Tensor]]:
|
| 23 |
+
"""Split incoming weight updates into parameter and buffer updates.
|
| 24 |
+
|
| 25 |
+
Returns the parameter updates, the buffer updates, and the model's
|
| 26 |
+
``named_buffers`` map so callers can reuse it without re-iterating.
|
| 27 |
+
"""
|
| 28 |
+
named_buffers = dict(model.named_buffers())
|
| 29 |
+
param_updates, buffer_updates = [], []
|
| 30 |
+
for name, tensor in weights:
|
| 31 |
+
if name in named_buffers:
|
| 32 |
+
buffer_updates.append((name, tensor))
|
| 33 |
+
else:
|
| 34 |
+
param_updates.append((name, tensor))
|
| 35 |
+
return param_updates, buffer_updates, named_buffers
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
@torch.no_grad()
|
| 39 |
+
def apply_buffer_updates(
|
| 40 |
+
model: torch.nn.Module,
|
| 41 |
+
buffer_updates: list[WeightUpdate],
|
| 42 |
+
named_buffers: dict[str, torch.Tensor] | None = None,
|
| 43 |
+
) -> int:
|
| 44 |
+
"""Copy updated buffer tensors into the target model in-place."""
|
| 45 |
+
if not buffer_updates:
|
| 46 |
+
return 0
|
| 47 |
+
|
| 48 |
+
if named_buffers is None:
|
| 49 |
+
named_buffers = dict(model.named_buffers())
|
| 50 |
+
loaded = 0
|
| 51 |
+
for name, tensor in buffer_updates:
|
| 52 |
+
if name not in named_buffers:
|
| 53 |
+
continue
|
| 54 |
+
|
| 55 |
+
target = named_buffers[name]
|
| 56 |
+
if target.shape != tensor.shape:
|
| 57 |
+
raise ValueError(
|
| 58 |
+
f"Buffer shape mismatch for {name}: expected {tuple(target.shape)}, got {tuple(tensor.shape)}"
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
source = tensor.to(device=target.device, dtype=target.dtype, non_blocking=False)
|
| 62 |
+
target.copy_(source, non_blocking=False)
|
| 63 |
+
loaded += 1
|
| 64 |
+
|
| 65 |
+
return loaded
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"additional_special_tokens": [
|
| 3 |
+
"<|IMAGE_PLACEHOLDER|>",
|
| 4 |
+
"<|image_pad|>",
|
| 5 |
+
"<|IMAGE_START|>",
|
| 6 |
+
"<|IMAGE_END|>",
|
| 7 |
+
"<|video_pad|>"
|
| 8 |
+
],
|
| 9 |
+
"bos_token": {
|
| 10 |
+
"content": "<s>",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"cls_token": {
|
| 17 |
+
"content": "<|begin_of_sentence|>",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
},
|
| 23 |
+
"eos_token": {
|
| 24 |
+
"content": "</s>",
|
| 25 |
+
"lstrip": false,
|
| 26 |
+
"normalized": false,
|
| 27 |
+
"rstrip": false,
|
| 28 |
+
"single_word": false
|
| 29 |
+
},
|
| 30 |
+
"mask_token": {
|
| 31 |
+
"content": "<mask:1>",
|
| 32 |
+
"lstrip": false,
|
| 33 |
+
"normalized": false,
|
| 34 |
+
"rstrip": false,
|
| 35 |
+
"single_word": false
|
| 36 |
+
},
|
| 37 |
+
"pad_token": {
|
| 38 |
+
"content": "<unk>",
|
| 39 |
+
"lstrip": false,
|
| 40 |
+
"normalized": false,
|
| 41 |
+
"rstrip": false,
|
| 42 |
+
"single_word": false
|
| 43 |
+
},
|
| 44 |
+
"sep_token": {
|
| 45 |
+
"content": "<|end_of_sentence|>",
|
| 46 |
+
"lstrip": false,
|
| 47 |
+
"normalized": false,
|
| 48 |
+
"rstrip": false,
|
| 49 |
+
"single_word": false
|
| 50 |
+
},
|
| 51 |
+
"unk_token": {
|
| 52 |
+
"content": "<unk>",
|
| 53 |
+
"lstrip": false,
|
| 54 |
+
"normalized": false,
|
| 55 |
+
"rstrip": false,
|
| 56 |
+
"single_word": false
|
| 57 |
+
}
|
| 58 |
+
}
|
tokenizer_config.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|