From ce186c84921a6cef0e8593994515e559a5d8101a Mon Sep 17 00:00:00 2001 From: UsernameFull Date: Tue, 23 Jun 2026 16:01:10 +0800 Subject: [PATCH 1/4] docs: update Ascend NPU documentation --- .github/workflows/ci-npu-test.yml | 6 +- docker/Dockerfile.A2 | 18 +- docker/Dockerfile.A3 | 18 +- .../Hardware Support/ascend_docker_usage.md | 103 ++- .../Hardware Support/ascend_npu_env_config.md | 27 +- .../Hardware Support/ascend_npu_examples.md | 233 +++-- .../Hardware Support/ascend_npu_faq.md | 35 +- .../Hardware Support/ascend_npu_rlvr.md | 236 ++--- .../Hardware Support/ascend_usage.md | 94 +- .../Hardware Support/ascend_docker_usage.md | 67 +- .../Hardware Support/ascend_npu_env_config.md | 27 +- .../Hardware Support/ascend_npu_examples.md | 233 +++-- .../Hardware Support/ascend_npu_faq.md | 35 +- .../Hardware Support/ascend_npu_rlvr.md | 822 ++++++++++++++++++ .../Hardware Support/ascend_usage.md | 141 +-- examples/ascend_examples/run_dpo_pipeline.sh | 5 - examples/ascend_examples/run_rlvr_pipeline.sh | 2 +- 17 files changed, 1624 insertions(+), 478 deletions(-) create mode 100644 docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_npu_rlvr.md delete mode 100644 examples/ascend_examples/run_dpo_pipeline.sh diff --git a/.github/workflows/ci-npu-test.yml b/.github/workflows/ci-npu-test.yml index 2a77b56cd..6f6034fad 100644 --- a/.github/workflows/ci-npu-test.yml +++ b/.github/workflows/ci-npu-test.yml @@ -198,7 +198,7 @@ jobs: - name: Install ROLL requirements run: | python3 -m pip install -r requirements_common.txt - python3 -m pip install deepspeed==0.16.4 tensorboard + python3 -m pip install tensorboard - name: Install SGLang NPU kernel from source shell: bash @@ -289,7 +289,7 @@ jobs: except metadata.PackageNotFoundError: return "not installed" - packages = ("vllm-ascend", "transformers", "deepspeed", "triton-ascend") + packages = ("vllm-ascend", "transformers", "triton-ascend") for package_name in packages: print(f"{package_name}={package_version(package_name)}") @@ -302,7 +302,6 @@ jobs: export PYTHONPATH="${GITHUB_WORKSPACE}:${PYTHONPATH:-}" python3 -m pytest tests/third_party/sglang \ tests/third_party/vllm \ - tests/third_party/deepspeed \ tests/distributed \ tests/models \ tests/pipeline \ @@ -315,4 +314,3 @@ jobs: -v --timeout=600 --durations=0 --durations-min=0 -x env: ROLL_NPU_CI: "1" - DS_UNITTEST_TIMEOUT: "600" diff --git a/docker/Dockerfile.A2 b/docker/Dockerfile.A2 index b5a74ae53..720512d08 100644 --- a/docker/Dockerfile.A2 +++ b/docker/Dockerfile.A2 @@ -1,4 +1,4 @@ -FROM quay.io/ascend/cann:8.5.1-910b-ubuntu22.04-py3.11 +FROM quay.io/ascend/cann:9.0.0-910b-ubuntu22.04-py3.11 ARG SOC_VERSION="ascend910b1" @@ -9,6 +9,7 @@ ENV PIP_CONSTRAINT="" ENV LD_LIBRARY_PATH=/usr/local/Ascend/driver/lib64:/usr/local/Ascend/driver/lib64/common:/usr/local/Ascend/driver/lib64/driver:/usr/local/Ascend/add-ons:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64:/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64:/usr/local/Ascend/ascend-toolkit/latest/atc/lib64:${LD_LIBRARY_PATH} ENV ASCEND_HOME_PATH=/usr/local/Ascend/ascend-toolkit/latest +ENV HCCL_NPU_SOCKET_PORT_RANGE=auto RUN apt-get update -y && \ apt-get install -y --no-install-recommends \ @@ -26,8 +27,8 @@ RUN pip config set global.index-url https://mirrors.huaweicloud.com/repository/p WORKDIR /workspace -RUN git clone --depth 1 -b v0.13.0 https://github.com/vllm-project/vllm.git && \ - git clone --depth 1 -b releases/v0.13.0 https://github.com/vllm-project/vllm-ascend.git +RUN git clone --depth 1 -b v0.18.0 https://github.com/vllm-project/vllm.git && \ + git clone --depth 1 -b v0.18.0 https://github.com/vllm-project/vllm-ascend.git RUN cd vllm && \ pip install -r requirements/build.txt && \ @@ -36,7 +37,7 @@ RUN cd vllm && \ pip cache purge && \ cd .. -RUN pip install torch==2.8.0+cpu torchvision==0.23.0 torchaudio==2.8.0 \ +RUN pip install torch==2.9.0+cpu torchvision==0.24.0 torchaudio==2.9.0 \ --index-url https://download.pytorch.org/whl/cpu RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ @@ -50,17 +51,18 @@ COPY . . RUN pip install -r requirements_common.txt -RUN pip install "deepspeed==0.16.4" "transformers==4.57.6" "tensorboard==2.20.0" +RUN pip install "transformers==4.57.6" "tensorboard==2.20.0" RUN pip install -e . RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ pip uninstall -y triton && \ pip uninstall -y triton-ascend && \ - pip install triton-ascend==3.2.0 && \ + pip install triton-ascend==3.2.1 --extra-index-url https://mirrors.huaweicloud.com/ascend/repos/pypi && \ pip cache purge RUN echo "source /usr/local/Ascend/ascend-toolkit/set_env.sh" >> /root/.bashrc && \ - echo "source /usr/local/Ascend/nnal/atb/set_env.sh" >> /root/.bashrc + echo "source /usr/local/Ascend/nnal/atb/set_env.sh" >> /root/.bashrc && \ + echo "export HCCL_NPU_SOCKET_PORT_RANGE=auto" >> /root/.bashrc -CMD ["/bin/bash"] \ No newline at end of file +CMD ["/bin/bash"] diff --git a/docker/Dockerfile.A3 b/docker/Dockerfile.A3 index c6762b57e..07838b8eb 100644 --- a/docker/Dockerfile.A3 +++ b/docker/Dockerfile.A3 @@ -1,4 +1,4 @@ -FROM quay.io/ascend/cann:8.5.1-a3-ubuntu22.04-py3.11 +FROM quay.io/ascend/cann:9.0.0-a3-ubuntu22.04-py3.11 ARG SOC_VERSION="ascend910_9391" @@ -9,6 +9,7 @@ ENV PIP_CONSTRAINT="" ENV LD_LIBRARY_PATH=/usr/local/Ascend/driver/lib64:/usr/local/Ascend/driver/lib64/common:/usr/local/Ascend/driver/lib64/driver:/usr/local/Ascend/add-ons:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64:/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64:/usr/local/Ascend/ascend-toolkit/latest/atc/lib64:${LD_LIBRARY_PATH} ENV ASCEND_HOME_PATH=/usr/local/Ascend/ascend-toolkit/latest +ENV HCCL_NPU_SOCKET_PORT_RANGE=auto RUN apt-get update -y && \ apt-get install -y --no-install-recommends \ @@ -26,8 +27,8 @@ RUN pip config set global.index-url https://mirrors.huaweicloud.com/repository/p WORKDIR /workspace -RUN git clone --depth 1 -b v0.13.0 https://github.com/vllm-project/vllm.git && \ - git clone --depth 1 -b releases/v0.13.0 https://github.com/vllm-project/vllm-ascend.git +RUN git clone --depth 1 -b v0.18.0 https://github.com/vllm-project/vllm.git && \ + git clone --depth 1 -b v0.18.0 https://github.com/vllm-project/vllm-ascend.git RUN cd vllm && \ pip install -r requirements/build.txt && \ @@ -36,7 +37,7 @@ RUN cd vllm && \ pip cache purge && \ cd .. -RUN pip install torch==2.8.0+cpu torchvision==0.23.0 torchaudio==2.8.0 \ +RUN pip install torch==2.9.0+cpu torchvision==0.24.0 torchaudio==2.9.0 \ --index-url https://download.pytorch.org/whl/cpu RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ @@ -50,17 +51,18 @@ COPY . . RUN pip install -r requirements_common.txt -RUN pip install "deepspeed==0.16.4" "transformers==4.57.6" "tensorboard==2.20.0" +RUN pip install "transformers==4.57.6" "tensorboard==2.20.0" RUN pip install -e . RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ pip uninstall -y triton && \ pip uninstall -y triton-ascend && \ - pip install triton-ascend==3.2.0 && \ + pip install triton-ascend==3.2.1 --extra-index-url https://mirrors.huaweicloud.com/ascend/repos/pypi && \ pip cache purge RUN echo "source /usr/local/Ascend/ascend-toolkit/set_env.sh" >> /root/.bashrc && \ - echo "source /usr/local/Ascend/nnal/atb/set_env.sh" >> /root/.bashrc + echo "source /usr/local/Ascend/nnal/atb/set_env.sh" >> /root/.bashrc && \ + echo "export HCCL_NPU_SOCKET_PORT_RANGE=auto" >> /root/.bashrc -CMD ["/bin/bash"] \ No newline at end of file +CMD ["/bin/bash"] diff --git a/docs_roll/docs/User Guides/Hardware Support/ascend_docker_usage.md b/docs_roll/docs/User Guides/Hardware Support/ascend_docker_usage.md index 259a77d66..e944aee50 100644 --- a/docs_roll/docs/User Guides/Hardware Support/ascend_docker_usage.md +++ b/docs_roll/docs/User Guides/Hardware Support/ascend_docker_usage.md @@ -1,8 +1,8 @@ # Running ROLL on Ascend NPU with Docker -Last updated: 04/27/2026. +Last updated: 06/23/2026. -This guide explains how to build and run ROLL on **Huawei Ascend NPU** using `Dockerfile.A2` and `Dockerfile.A3`. +This guide explains how to get, build, and run ROLL images on **Huawei Ascend NPU**. Prefer the pre-built image when possible; use `Dockerfile.A2` or `Dockerfile.A3` when you need to customize dependencies. Atlas A5 currently follows the manual installation profile in [ROLL x Ascend](ascend_usage.md). ## Hardware & Software Requirements @@ -10,32 +10,64 @@ This guide explains how to build and run ROLL on **Huawei Ascend NPU** using `Do | ---- | ------------- | ------------- | | Hardware | Atlas 900 A2 PODc (Ascend 910B1) | Atlas 900 A3 PODc (Ascend 910_9391) | | Host OS | Ubuntu 22.04 | Ubuntu 22.04 | -| CANN | 8.5.1 | 8.5.1 | +| CANN | 9.0.0 | 9.0.0 | | Python | 3.11 | 3.11 | | Docker | >= 20.10 | >= 20.10 | | Ascend NPU Driver | Installed on host | Installed on host | +This Docker guide covers the A2/A3 Dockerfiles. For Atlas A5, use the manual installation profile: torch 2.10, vLLM v0.20.2, vLLM-Ascend `main`, and `COMPILE_CUSTOM_KERNELS=1` when building vLLM-Ascend. + ## Key Components Both Dockerfiles install the same versions of core dependencies: | Component | Version | | --------- | ------- | -| PyTorch | 2.8.0+cpu | -| vLLM | 0.13.0 | -| vLLM-Ascend | 0.13.0 | -| DeepSpeed | 0.16.4 | +| PyTorch | 2.9.0+cpu | +| vLLM | 0.18.0 | +| vLLM-Ascend | 0.18 | | Transformers | 4.57.6 | -| triton-ascend | 3.2.0 | +| triton-ascend | 3.2.1 | + +Atlas A5 uses a newer manual installation stack: + +| Component | Atlas A5 Version / Setting | +| --------- | -------------------------- | +| PyTorch | 2.10 | +| vLLM | v0.20.2 | +| vLLM-Ascend | `main` branch | +| Required build variable | `COMPILE_CUSTOM_KERNELS=1` | The primary difference is the base image and SOC version: | Item | Dockerfile.A2 | Dockerfile.A3 | | ---- | ------------- | ------------- | -| Base Image | `quay.io/ascend/cann:8.5.1-910b-ubuntu22.04-py3.11` | `quay.io/ascend/cann:8.5.1-a3-ubuntu22.04-py3.11` | +| Base Image | `quay.io/ascend/cann:9.0.0-910b-ubuntu22.04-py3.11` | `quay.io/ascend/cann:9.0.0-a3-ubuntu22.04-py3.11` | | SOC_VERSION | `ascend910b1` | `ascend910_9391` | -## Build the Docker Image +## Get the Docker Image + +### Option A: Use the Pre-built Image (Recommended) + +Pull the image that matches your hardware, then tag it with the local name used by the commands below: + +**For Atlas 900 A2 PODc (Ascend 910B1):** + +```bash +docker pull quay.io/ascend/roll:main-a2 +docker tag quay.io/ascend/roll:main-a2 roll:ascend-a2 +``` + +**For Atlas 900 A3 PODc (Ascend 910_9391):** + +```bash +docker pull quay.io/ascend/roll:main-a3 +docker tag quay.io/ascend/roll:main-a3 roll:ascend-a3 +``` + +Check https://quay.io/repository/ascend/roll?tab=tags for available image tags. If you use a pre-built image, continue with [Run the Container](#run-the-container). + +### Option B: Build from Dockerfile ### 1. Clone the ROLL Repository @@ -88,6 +120,7 @@ docker run -dit \ --device /dev/davinci4 \ --device /dev/davinci5 \ --device /dev/davinci6 \ + --device /dev/davinci7 \ --device /dev/davinci_manager \ --device /dev/devmm_svm \ --device /dev/hisi_hdc \ @@ -115,6 +148,7 @@ docker run -dit \ --device /dev/davinci4 \ --device /dev/davinci5 \ --device /dev/davinci6 \ + --device /dev/davinci7 \ --device /dev/davinci_manager \ --device /dev/devmm_svm \ --device /dev/hisi_hdc \ @@ -130,6 +164,44 @@ docker run -dit \ /bin/bash ``` +### Multi-NPU Startup (Recommended for Training) + +For multi-NPU training, mount all available NPU devices. Adjust the number of `--device /dev/davinciX` entries according to the NPU count on your node: + +```bash +docker run -dit \ + --name roll_ascend \ + --device /dev/davinci0 \ + --device /dev/davinci1 \ + --device /dev/davinci2 \ + --device /dev/davinci3 \ + --device /dev/davinci4 \ + --device /dev/davinci5 \ + --device /dev/davinci6 \ + --device /dev/davinci7 \ + --device /dev/davinci_manager \ + --device /dev/devmm_svm \ + --device /dev/hisi_hdc \ + -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ + -v /usr/local/Ascend/add-ons:/usr/local/Ascend/add-ons \ + -v /usr/local/dcmi:/usr/local/dcmi \ + -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \ + -v /etc/ascend_install.info:/etc/ascend_install.info \ + -v /home/$USER:/home/$USER \ + -v /path/to/models:/path/to/models \ + -v /path/to/data:/path/to/data \ + --ipc=host \ + --net=host \ + roll:ascend-a3 \ + /bin/bash +``` + +> **Note:** +> - `--device /dev/davinciX`: Mounts NPU devices. Add or remove entries based on available NPU count. +> - `--device /dev/davinci_manager`, `--device /dev/devmm_svm`, `--device /dev/hisi_hdc`: Required management devices for Ascend NPU. +> - `-v /usr/local/Ascend/driver`: Mounts the host Ascend driver. +> - `-v /path/to/models` and `-v /path/to/data`: Mount model weights and training data directories as needed. + ### Enter the Container ```bash @@ -152,7 +224,7 @@ npu-smi info env | grep -E "ASCEND|LD_LIBRARY_PATH|PATH" # Verify Python packages -python -c "import torch; import torch_npu; print(torch_npu.npu.is_available())" +python -c "import torch; import torch_npu; print(torch.npu.is_available())" python -c "import vllm; print(f'vllm: {vllm.__version__}')" python -c "import vllm_ascend; print(f'vllm_ascend available')" ``` @@ -161,10 +233,9 @@ python -c "import vllm_ascend; print(f'vllm_ascend available')" ### Important Configuration Notes -Since Megatron-LM training is not yet supported on Ascend NPU, you need to use **DeepSpeed** as the training backend. Make sure your configuration files use the following settings: +Since Megatron-LM is not supported on Ascend NPU, you need to use **FSDP2** as the training backend. Make sure your configuration files use the following settings: -1. Set `strategy_args` to use DeepSpeed -2. Set `device_mapping` to ensure training and inference are performed on different NPUs +1. Set `strategy_args` to use FSDP2 ### Example: RLVR Pipeline @@ -172,9 +243,11 @@ Since Megatron-LM training is not yet supported on Ascend NPU, you need to use * # After modifying model paths and adjusting device_mapping python examples/start_rlvr_pipeline.py \ --config_path ascend_examples \ - --config_name qwen3_8b_rlvr_deepspeed + --config_name qwen3_30b_rlvr_fsdp2 ``` +> **Note:** The `qwen3_30b_rlvr_fsdp2` configuration is specifically designed for Ascend NPU with FSDP2 as the training backend. Adjust `device_mapping` in the configuration file according to your NPU topology. + ## Troubleshooting ### NPU Not Visible Inside Container diff --git a/docs_roll/docs/User Guides/Hardware Support/ascend_npu_env_config.md b/docs_roll/docs/User Guides/Hardware Support/ascend_npu_env_config.md index 219b3b654..5cba8754e 100644 --- a/docs_roll/docs/User Guides/Hardware Support/ascend_npu_env_config.md +++ b/docs_roll/docs/User Guides/Hardware Support/ascend_npu_env_config.md @@ -1,6 +1,6 @@ # Ascend NPU Environment Configuration Guide -Last updated: 04/27/2026. +Last updated: 06/23/2026. This document describes the key environment variables for running ROLL on Huawei Ascend NPU, covering device management, HCCL communication, memory optimization, CPU scheduling, vLLM-Ascend inference, and debugging. @@ -79,6 +79,7 @@ These variables control the behavior of HCCL (Huawei Collective Communication Li | `HCCL_DETERMINISTIC` | `false` | Disable deterministic computation. Enabling it significantly reduces communication performance | | `HCCL_OP_EXPANSION_MODE` | `"AIV"` | Communication algorithm dispatch location. `AIV` uses Vector Core, outperforms `AI_CPU`/`HOST`/`HOST_TS` | | `HCCL_BUFFSIZE` | e.g. `"2147483648"` | HCCL communication buffer size in bytes. Increase for large data volume scenarios | +| `HCCL_NPU_SOCKET_PORT_RANGE` | `auto` | Allow HCCL to allocate non-default device-side NIC ports when multiple worker processes run on the same NPU | | `HCCL_IF_IP` | Node's IP address | Specify the IP address used by HCCL for inter-node communication. Required for multi-node training | | `HCCL_SOCKET_IFNAME` | e.g. `"enp194s0f0"` | Network interface name for HCCL socket communication. Must be consistent across all nodes | | `HCCL_IF_BASE_PORT` | e.g. `23456` | Base port for HCCL inter-node communication. Ensure ports are not blocked by firewall | @@ -90,6 +91,7 @@ Example (single-node): export HCCL_CONNECT_TIMEOUT=3600 export HCCL_DETERMINISTIC=false export HCCL_OP_EXPANSION_MODE="AIV" +export HCCL_NPU_SOCKET_PORT_RANGE="auto" ``` Example (multi-node): @@ -99,6 +101,7 @@ export HCCL_CONNECT_TIMEOUT=3600 export HCCL_EXEC_TIMEOUT=3600 export HCCL_DETERMINISTIC=false export HCCL_OP_EXPANSION_MODE="AIV" +export HCCL_NPU_SOCKET_PORT_RANGE="auto" export HCCL_IF_IP=$(hostname -I | awk '{print $1}') export HCCL_SOCKET_IFNAME="enp194s0f0" export HCCL_IF_BASE_PORT=23456 @@ -151,8 +154,7 @@ export CPU_AFFINITY_CONF=1,npu0:0-1,npu1:2-3,npu2:4-5,npu3:6-7 | `VLLM_USE_V1` | `1` | Enable vLLM V1 architecture. Required for vLLM-Ascend | | `VLLM_ATTENTION_BACKEND` | `XFORMERS` | vLLM attention computation backend | | `VLLM_ASCEND_ENABLE_FLASHCOMM` | `1` | Enable Ascend FlashComm high-speed communication optimization | -| `VLLM_ASCEND_ENABLE_DENSE_OPTIMIZE` | `1` | Enable dense computation optimization for large model inference | -| `VLLM_ASCEND_ENABLE_PREFETCH_MLP` | `1` | Enable MLP layer weight prefetching | +| `VLLM_ASCEND_ENABLE_PREFETCH_MLP` | `1` | Enable MLP layer weight prefetching. This replaces the older dense optimize toggle in current vLLM-Ascend releases. | | `VLLM_ASCEND_ENABLE_TOPK_OPTIMIZE` | `1` | Enable TopK operator fusion optimization for generation decoding | | `VLLM_ASCEND_MODEL_EXECUTE_TIME_OBSERVE` | `1` | Print prefill/decode phase timing details (for debugging) | | `VLLM_ASCEND_TRACE_RECOMPILES` | `1` | Trace operator recompilation for debugging performance issues | @@ -164,10 +166,26 @@ Example: export VLLM_USE_V1=1 export VLLM_ATTENTION_BACKEND=XFORMERS export VLLM_ASCEND_ENABLE_FLASHCOMM=1 -export VLLM_ASCEND_ENABLE_DENSE_OPTIMIZE=1 export VLLM_ASCEND_ENABLE_PREFETCH_MLP=1 ``` +## vLLM-Ascend Build Variables + +The following variable is used when building vLLM-Ascend from source. Set it before `pip install -e .`; it does not need to be exported for every ROLL run. + +| Variable | Recommended Value | Description | +| -------- | ----------------- | ----------- | +| `COMPILE_CUSTOM_KERNELS` | `1` for Atlas A5 | Compile vLLM-Ascend custom kernels. Required by the Atlas A5 installation profile that uses vLLM-Ascend `main`. | + +Example (Atlas A5): + +```bash +git clone -b main --depth 1 https://github.com/vllm-project/vllm-ascend.git +cd vllm-ascend +export COMPILE_CUSTOM_KERNELS=1 +pip install -v -e . +``` + ## CANN Logging & Debugging Variables | Variable | Recommended Value | Description | @@ -243,7 +261,6 @@ export OMP_NUM_THREADS=1 # vLLM-Ascend inference export VLLM_USE_V1=1 export VLLM_ASCEND_ENABLE_FLASHCOMM=1 -export VLLM_ASCEND_ENABLE_DENSE_OPTIMIZE=1 export VLLM_ASCEND_ENABLE_PREFETCH_MLP=1 # Operator compilation cache diff --git a/docs_roll/docs/User Guides/Hardware Support/ascend_npu_examples.md b/docs_roll/docs/User Guides/Hardware Support/ascend_npu_examples.md index 22385bcd7..60b4be3d6 100644 --- a/docs_roll/docs/User Guides/Hardware Support/ascend_npu_examples.md +++ b/docs_roll/docs/User Guides/Hardware Support/ascend_npu_examples.md @@ -8,11 +8,11 @@ This document provides end-to-end configuration examples for running ROLL on Hua Before running these examples, ensure you have: -1. Pulled the pre-built Ascend image that matches your hardware (see [Docker Usage Guide](ascend_docker_usage.md)). +1. Pulled or built the Ascend image that matches your hardware (see [Docker Usage Guide](ascend_docker_usage.md)). 2. Verified the environment inside the container (see [Verify the Environment](ascend_docker_usage.md#verify-the-environment)). 3. Downloaded the model weights to a directory accessible from inside the container. -The repository currently includes a runnable Ascend RLVR example in `examples/ascend_examples`, including `qwen3_8b_rlvr_deepspeed.yaml` and `run_rlvr_pipeline.sh`. +The repository currently includes a runnable Ascend RLVR example in `examples/ascend_examples`, including `qwen3_30b_rlvr_fsdp2.yaml` and `run_rlvr_pipeline.sh`. ## Key Differences from GPU @@ -20,15 +20,14 @@ When adapting GPU configurations for NPU, the following changes are **required** | Item | GPU | NPU | | ---- | --- | --- | -| Training backend | Megatron or DeepSpeed | DeepSpeed only (Megatron not supported) | -| Device placement | Colocated mode supported | Colocated mode **not** supported; training and inference must use separate NPUs | +| Training backend | Megatron or FSDP2 | FSDP2 only (Megatron not supported on NPU) | | Attention implementation | `flash_attn` or `fa2` | `fa2` via `transformers` (not `flash_attn` package) | | Communication backend | NCCL | HCCL | | Device visibility | `CUDA_VISIBLE_DEVICES` | `ASCEND_RT_VISIBLE_DEVICES` | ## Example 1: Single-Node Agentic Pipeline (Qwen2.5-0.5B) -This example runs the FrozenLake agentic pipeline on a single 8-NPU node using DeepSpeed ZeRO-3. +This example runs the FrozenLake agentic pipeline on a single 8-NPU node using FSDP2. ### Step 1: Start the Container @@ -67,6 +66,7 @@ docker run -dit \ export HCCL_CONNECT_TIMEOUT=3600 export HCCL_DETERMINISTIC=false export HCCL_OP_EXPANSION_MODE="AIV" +export HCCL_NPU_SOCKET_PORT_RANGE="auto" # NPU memory export NPU_MEMORY_FRACTION=0.96 @@ -81,8 +81,8 @@ export OMP_NUM_THREADS=1 # vLLM-Ascend inference export VLLM_USE_V1=1 +export VLLM_ASCEND_ENABLE_NZ=0 export VLLM_ASCEND_ENABLE_FLASHCOMM=1 -export VLLM_ASCEND_ENABLE_DENSE_OPTIMIZE=1 export VLLM_ASCEND_ENABLE_PREFETCH_MLP=1 # Operator compilation cache @@ -98,15 +98,11 @@ export ATB_LOG_LEVEL=ERROR ### Step 3: Create NPU Configuration File -Create a YAML config file (e.g., `agentic_frozen_lake_npu.yaml`) with the following NPU-specific settings. Key differences from the GPU config are marked with `# NPU` comments: +Create `examples/agentic_frozen_lake_npu/agentic_frozen_lake_npu.yaml` with the following NPU-specific settings. This location matters: the config uses `defaults: - ../config/traj_envs@_here_`, which depends on the relative directory layout under `examples/`. If you save the file elsewhere, update the `defaults` path accordingly. Key differences from the GPU config are marked with `# NPU` comments: ```yaml defaults: - ../config/traj_envs@_here_ - - ../config/deepspeed_zero@_here_ - - ../config/deepspeed_zero2@_here_ - - ../config/deepspeed_zero3@_here_ - - ../config/deepspeed_zero3_cpuoffload@_here_ hydra: run: @@ -123,6 +119,8 @@ system_envs: HCCL_CONNECT_TIMEOUT: "3600" HCCL_DETERMINISTIC: "false" HCCL_OP_EXPANSION_MODE: "AIV" + HCCL_NPU_SOCKET_PORT_RANGE: "auto" + VLLM_ASCEND_ENABLE_NZ: "0" NPU_MEMORY_FRACTION: "0.96" CPU_AFFINITY_CONF: "2" OMP_NUM_THREADS: "1" @@ -171,8 +169,13 @@ actor_train: data_args: template: qwen2_5 strategy_args: - strategy_name: deepspeed_train # NPU: Must use DeepSpeed, NOT megatron_train - strategy_config: ${deepspeed_zero3} + strategy_name: fsdp2_train + strategy_config: + fsdp_size: 4 + param_dtype: bf16 + reduce_dtype: bf16 + reshard_after_forward: true + offload_policy: false # NPU: Must use FSDP2, NOT megatron_train device_mapping: list(range(0,4)) # NPU: Training on NPUs 0-3 infer_batch_size: 2 @@ -207,8 +210,13 @@ reference: data_args: template: qwen2_5 strategy_args: - strategy_name: hf_infer - strategy_config: ~ + strategy_name: fsdp2_infer + strategy_config: + fsdp_size: 4 + param_dtype: bf16 + reduce_dtype: bf16 + reshard_after_forward: true + offload_policy: false device_mapping: list(range(4,8)) # NPU: Share inference NPUs with actor_infer infer_batch_size: 2 @@ -252,13 +260,13 @@ cd /workspace/ROLL export PYTHONPATH="/workspace/ROLL:$PYTHONPATH" python examples/start_agentic_pipeline.py \ - --config_path \ + --config_path agentic_frozen_lake_npu \ --config_name agentic_frozen_lake_npu ``` -## Example 2: Single-Node RLVR Pipeline (Qwen3-8B) +## Example 2: Single-Node RLVR Pipeline (Qwen3-30B-A3B) -This example runs the RLVR pipeline on Ascend NPU using the repository config `examples/ascend_examples/qwen3_8b_rlvr_deepspeed.yaml`. +This example runs the RLVR pipeline on Ascend NPU using the repository config `examples/ascend_examples/qwen3_30b_rlvr_fsdp2.yaml`. ### Key Configuration Changes @@ -268,6 +276,8 @@ system_envs: HCCL_CONNECT_TIMEOUT: "3600" HCCL_DETERMINISTIC: "false" HCCL_OP_EXPANSION_MODE: "AIV" + HCCL_NPU_SOCKET_PORT_RANGE: "auto" + VLLM_ASCEND_ENABLE_NZ: "0" NPU_MEMORY_FRACTION: "0.96" CPU_AFFINITY_CONF: "2" OMP_NUM_THREADS: "1" @@ -276,26 +286,25 @@ system_envs: rollout_batch_size: 32 prompt_length: 2048 -response_length: 8192 +response_length: 4096 num_return_sequences_in_group: 8 -pretrain: Qwen/Qwen3-8B-Base -reward_pretrain: Qwen/Qwen3-8B-Base +pretrain: Qwen/Qwen3-30B-A3B +reward_pretrain: Qwen/Qwen3-30B-A3B actor_train: model_args: - attn_implementation: fa2 # NPU: Use fa2 via transformers, NOT flash_attn disable_gradient_checkpointing: false dtype: bf16 model_type: ~ training_args: learning_rate: 1.0e-6 weight_decay: 0 - per_device_train_batch_size: 1 - gradient_accumulation_steps: 32 + per_device_train_batch_size: 2 + gradient_accumulation_steps: 8 warmup_steps: 20 data_args: - template: qwen3 + template: qwen2_5 file_name: - data/math_deepmath_deal.jsonl domain_interleave_probs: @@ -303,10 +312,28 @@ actor_train: dataset_dir: data messages: messages interleave_probs: "1.0" + preprocessing_num_workers: 16 strategy_args: - strategy_name: deepspeed_train # NPU: Must use DeepSpeed - strategy_config: ${deepspeed_zero3} - device_mapping: list(range(0,8)) # NPU: Training on NPUs 0-7 + strategy_name: fsdp2_train + strategy_config: + fsdp_size: 16 + param_dtype: bf16 + reduce_dtype: bf16 + offload_policy: true + apply_expert_patch: true # NPU: Required for MoE models + apply_tiled_mlp: true # NPU: TiledMLP to reduce memory + tiled_num_shards: 8 + reshard_after_forward: true + wrap_policy: # NPU: MoE-specific wrap policy + wrap_embeddings: true + wrap_lm_output: true + moe_experts: + - Qwen3MoeMLP + transformer_layer_cls_to_wrap: + - Qwen3MoeAttention + - Qwen3MoeSparseMoeBlock + use_remove_padding: true + device_mapping: list(range(0,16)) # NPU: Training on NPUs 0-15 infer_batch_size: 2 actor_infer: @@ -321,15 +348,18 @@ actor_infer: temperature: 0.99 num_return_sequences: ${num_return_sequences_in_group} data_args: - template: qwen3 + template: qwen2_5 strategy_args: strategy_name: vllm strategy_config: gpu_memory_utilization: 0.8 block_size: 16 - max_model_len: 8000 - device_mapping: list(range(8,12)) # NPU: Inference on NPUs 8-11 - infer_batch_size: 4 + max_model_len: 6144 + tensor_parallel_size: 2 + enforce_eager: true + load_format: dummy + device_mapping: list(range(0,16)) # NPU: Inference shares NPUs with training + infer_batch_size: 1 reference: model_args: @@ -337,12 +367,19 @@ reference: dtype: bf16 model_type: ~ data_args: - template: qwen3 + template: qwen2_5 strategy_args: - strategy_name: hf_infer - strategy_config: ~ - device_mapping: list(range(12,16)) # NPU: Reference on NPUs 12-15 - infer_batch_size: 1 + strategy_name: fsdp2_infer + strategy_config: + fsdp_size: 16 + param_dtype: bf16 + reduce_dtype: bf16 + apply_tiled_mlp: true + tiled_num_shards: 8 + reshard_after_forward: true + offload_policy: true + device_mapping: list(range(0,16)) # NPU: Reference shares NPUs with training + infer_batch_size: 2 rewards: math_rule: @@ -350,8 +387,8 @@ rewards: model_args: model_name_or_path: ${reward_pretrain} data_args: - template: qwen3 - tag_included: [deepmath_103k, MATH-500, OlympiadBench, minervamath, aime2025, gsm8k, aime, amc23, math_rule] + template: qwen2_5 + tag_included: [deepmath_103k, aime] world_size: 8 infer_batch_size: 1 ``` @@ -364,7 +401,7 @@ export PYTHONPATH="/workspace/ROLL:$PYTHONPATH" python examples/start_rlvr_pipeline.py \ --config_path ascend_examples \ - --config_name qwen3_8b_rlvr_deepspeed + --config_name qwen3_30b_rlvr_fsdp2 ``` ## Example 3: Multi-Node Distributed Training @@ -417,7 +454,7 @@ Before starting, identify the correct HCCL network interface on each node: ip addr # Or use the NPU tool to check HCCL interfaces -for i in {0..7}; do hccn_tool -i $i -ip -g; done +for i in $(seq 0 7); do hccn_tool -i $i -ip -g; done # The NPU device IPs are typically on a high-speed interconnect (e.g., 192.168.x.x). # Use the corresponding ethernet interface name (e.g., enp194s0f0, eth0) for HCCL_SOCKET_IFNAME. @@ -462,16 +499,16 @@ On **each** node, verify that NPU devices can communicate: ```bash # Check link status (all should show "up") -for i in {0..7}; do hccn_tool -i $i -link -g; done +for i in $(seq 0 7); do hccn_tool -i $i -link -g; done # Check TLS consistency (all should show the same switch value) -for i in {0..7}; do hccn_tool -i $i -tls -g; done | grep switch +for i in $(seq 0 7); do hccn_tool -i $i -tls -g; done | grep switch # If TLS is inconsistent, disable it on all cards on all nodes: -for i in {0..7}; do hccn_tool -i $i -tls -s enable 0; done +for i in $(seq 0 7); do hccn_tool -i $i -tls -s enable 0; done # Check NPU device IPs -for i in {0..7}; do hccn_tool -i $i -ip -g; done +for i in $(seq 0 7); do hccn_tool -i $i -ip -g; done # Test cross-node connectivity (run on node B, replace with node A's device IP) hccn_tool -i 0 -ping -g address @@ -494,6 +531,7 @@ export HCCL_CONNECT_TIMEOUT=3600 export HCCL_EXEC_TIMEOUT=3600 export HCCL_DETERMINISTIC=false export HCCL_OP_EXPANSION_MODE="AIV" +export HCCL_NPU_SOCKET_PORT_RANGE="auto" export HCCL_IF_IP= # Current node's IP address export HCCL_SOCKET_IFNAME= # e.g., enp194s0f0 export HCCL_IF_BASE_PORT=23456 @@ -511,8 +549,8 @@ export OMP_NUM_THREADS=1 # === vLLM-Ascend inference === export VLLM_USE_V1=1 +export VLLM_ASCEND_ENABLE_NZ=0 export VLLM_ASCEND_ENABLE_FLASHCOMM=1 -export VLLM_ASCEND_ENABLE_DENSE_OPTIMIZE=1 export VLLM_ASCEND_ENABLE_PREFETCH_MLP=1 # === Operator compilation cache === @@ -619,8 +657,13 @@ num_gpus_per_node: 8 # Training on Node0 NPUs 0-7 actor_train: strategy_args: - strategy_name: deepspeed_train - strategy_config: ${deepspeed_zero3_cpuoffload} + strategy_name: fsdp2_train + strategy_config: + fsdp_size: 8 + param_dtype: bf16 + reduce_dtype: bf16 + reshard_after_forward: true + offload_policy: true device_mapping: list(range(0,8)) # Inference on Node1 NPUs 0-7 @@ -636,28 +679,32 @@ actor_infer: # Reference model shares inference NPUs reference: strategy_args: - strategy_name: hf_infer - strategy_config: ~ + strategy_name: fsdp2_infer + strategy_config: + fsdp_size: 8 + param_dtype: bf16 + reduce_dtype: bf16 + reshard_after_forward: true + offload_policy: true device_mapping: list(range(8,16)) ``` Complete multi-node RLVR config example (2 nodes × 8 NPUs): ```yaml -defaults: - - ../config/deepspeed_zero@_here_ - - ../config/deepspeed_zero3@_here_ - - ../config/deepspeed_zero3_cpuoffload@_here_ - hydra: run: dir: . output_subdir: null -exp_name: "qwen2.5-7B-rlvr-npu-multinode" +exp_name: "qwen3-30BA3B-rlvr-npu-multinode" seed: 42 logging_dir: /data/logs output_dir: /data/output +system_envs: + USE_MODELSCOPE: '1' + HCCL_NPU_SOCKET_PORT_RANGE: "auto" + VLLM_ASCEND_ENABLE_NZ: "0" checkpoint_config: type: file_system @@ -671,7 +718,7 @@ logging_steps: 1 eval_steps: 10 resume_from_checkpoint: false -rollout_batch_size: 64 +rollout_batch_size: 32 prompt_length: 2048 response_length: 4096 num_return_sequences_in_group: 8 @@ -680,37 +727,52 @@ ppo_epochs: 1 adv_estimator: "reinforce" whiten_advantages: true -pretrain: /data/models/Qwen2.5-7B -reward_pretrain: /data/models/Qwen2.5-7B +pretrain: Qwen/Qwen3-30B-A3B +reward_pretrain: Qwen/Qwen3-30B-A3B actor_train: model_args: - attn_implementation: fa2 disable_gradient_checkpointing: false dtype: bf16 model_type: ~ training_args: learning_rate: 1.0e-6 weight_decay: 0 - per_device_train_batch_size: 1 - gradient_accumulation_steps: 32 + per_device_train_batch_size: 2 + gradient_accumulation_steps: 8 warmup_steps: 20 data_args: template: qwen2_5 file_name: - data/math_deepmath_deal.jsonl - - data/code_KodCode_data.jsonl domain_interleave_probs: - math_rule: 0.5 - code_sandbox: 0.5 + math_rule: 1 dataset_dir: /data/datasets messages: messages interleave_probs: "1.0" + preprocessing_num_workers: 16 strategy_args: - strategy_name: deepspeed_train - strategy_config: ${deepspeed_zero3_cpuoffload} + strategy_name: fsdp2_train + strategy_config: + fsdp_size: 8 + param_dtype: bf16 + reduce_dtype: bf16 + offload_policy: true + apply_expert_patch: true + apply_tiled_mlp: true + tiled_num_shards: 8 + reshard_after_forward: true + wrap_policy: + wrap_embeddings: true + wrap_lm_output: true + moe_experts: + - Qwen3MoeMLP + transformer_layer_cls_to_wrap: + - Qwen3MoeAttention + - Qwen3MoeSparseMoeBlock + use_remove_padding: true device_mapping: list(range(0,8)) # Node0 NPUs 0-7 for training - infer_batch_size: 4 + infer_batch_size: 2 actor_infer: model_args: @@ -730,23 +792,32 @@ actor_infer: strategy_config: gpu_memory_utilization: 0.8 block_size: 16 - max_model_len: 8000 + max_model_len: 6144 + tensor_parallel_size: 2 + enforce_eager: true + load_format: dummy device_mapping: list(range(8,16)) # Node1 NPUs 0-7 for inference infer_batch_size: 1 reference: model_args: - attn_implementation: fa2 disable_gradient_checkpointing: true dtype: bf16 model_type: ~ data_args: template: qwen2_5 strategy_args: - strategy_name: hf_infer - strategy_config: ~ + strategy_name: fsdp2_infer + strategy_config: + fsdp_size: 8 + param_dtype: bf16 + reduce_dtype: bf16 + apply_tiled_mlp: true + tiled_num_shards: 8 + reshard_after_forward: true + offload_policy: true device_mapping: list(range(8,16)) # Share inference NPUs - infer_batch_size: 8 + infer_batch_size: 2 rewards: math_rule: @@ -756,17 +827,7 @@ rewards: data_args: template: qwen2_5 tag_included: [deepmath_103k, aime] - world_size: 4 - infer_batch_size: 1 - code_sandbox: - use_local: true - worker_cls: roll.pipeline.rlvr.rewards.code_sandbox_reward_worker.CodeSandboxRewardWorker - tag_included: [KodCode] - model_args: - model_name_or_path: ${reward_pretrain} - data_args: - template: qwen2_5 - world_size: 4 + world_size: 8 infer_batch_size: 1 ``` @@ -797,7 +858,7 @@ Pattern 1 has lower cross-node HCCL communication overhead during inference. Pat ## Device Mapping Reference -Since NPU does not support colocated mode, you must allocate separate NPUs for training and inference. Here are common allocation patterns: +Here are common NPU allocation patterns. You may use colocated mode (training and inference sharing NPUs) or separate NPUs based on your workload and hardware: ### 8-NPU Single Node @@ -836,7 +897,7 @@ The first inference request after model loading triggers operator compilation, w If you encounter OOM with a 7B model on 4 NPUs: -1. Switch to `deepspeed_zero3_cpuoffload` strategy. +1. Switch to `fsdp2_train` strategy with `offload_policy: true`. 2. Reduce `per_device_train_batch_size` to 1. 3. Increase `gradient_accumulation_steps` accordingly. 4. Reduce `max_model_len` in vLLM config (e.g., from 8192 to 4096). diff --git a/docs_roll/docs/User Guides/Hardware Support/ascend_npu_faq.md b/docs_roll/docs/User Guides/Hardware Support/ascend_npu_faq.md index ea49f57a9..82122525d 100644 --- a/docs_roll/docs/User Guides/Hardware Support/ascend_npu_faq.md +++ b/docs_roll/docs/User Guides/Hardware Support/ascend_npu_faq.md @@ -88,33 +88,20 @@ export VLLM_ASCEND_ENABLE_NZ=0 ```bash pip uninstall -y triton triton-ascend -pip install triton-ascend==3.2.0 +pip install triton-ascend==3.2.1 --extra-index-url https://mirrors.huaweicloud.com/ascend/repos/pypi ``` ## Training Configuration -### Colocated Mode Not Supported - -**Symptom:** Training fails when `actor_train` and `actor_infer` share the same NPU devices. - -**Solution:** NPU does not support colocated mode. You must configure `device_mapping` so that training and inference run on separate NPUs. For example: - -```yaml -actor_train: - device_mapping: list(range(0, 4)) -actor_infer: - device_mapping: list(range(4, 8)) -``` - ### Megatron Strategy Not Supported **Symptom:** Errors when using `strategy: megatron` in configuration on NPU. -**Solution:** Megatron-LM training is not yet supported on Ascend NPU in the provided examples. Use DeepSpeed as the training backend: +**Solution:** Megatron-LM is not supported on Ascend NPU. Use FSDP2 as the training backend: ```yaml strategy_args: - strategy_name: deepspeed_train + strategy_name: fsdp2_train ``` ### HCCL Communication Timeout or Failure @@ -274,13 +261,6 @@ To make it persistent, add the following line to `/etc/security/limits.conf` ins * hard nofile 65536 ``` -You can also configure it globally in your ROLL YAML config: - -```yaml -system_envs: - RAY_ULIMIT_NOFILE: "65536" -``` - ### Out of NPU Memory **Symptom:** Training or inference crashes with OOM (Out of Memory) errors. @@ -289,11 +269,12 @@ system_envs: 1. Reduce `rollout_batch_size` or `num_return_sequences_in_group` in your configuration file. 2. Reduce `per_device_train_batch_size` and increase `gradient_accumulation_steps` accordingly. -3. Enable DeepSpeed ZeRO-3 with CPU offloading in your config: +3. Enable FSDP2 with CPU offloading in your config: ```yaml strategy_args: - strategy_name: deepspeed_train - strategy_config: ${deepspeed_zero3_cpuoffload} + strategy_name: fsdp2_train + strategy_config: + offload_policy: true ``` 4. Use a smaller model or apply LoRA to reduce memory footprint. @@ -303,7 +284,7 @@ system_envs: **Solution:** -1. Ensure CANN and vLLM-Ascend versions are compatible (both should be v0.13.0). +1. Ensure CANN and vLLM-Ascend versions are compatible (current images use CANN 9.0.0 with vLLM-Ascend v0.18). 2. Check that the SOC version matches your hardware. 3. Adjust vLLM parameters such as `gpu_memory_utilization` and `max_model_len` in your config. 4. Verify that `triton-ascend` is installed (not `triton`), as the wrong triton backend can cause kernel compilation fallbacks. diff --git a/docs_roll/docs/User Guides/Hardware Support/ascend_npu_rlvr.md b/docs_roll/docs/User Guides/Hardware Support/ascend_npu_rlvr.md index d51d57bea..63b520965 100644 --- a/docs_roll/docs/User Guides/Hardware Support/ascend_npu_rlvr.md +++ b/docs_roll/docs/User Guides/Hardware Support/ascend_npu_rlvr.md @@ -22,7 +22,7 @@ Ensure your hardware and host drivers are ready: | ---- | ----------- | | Hardware | Atlas 900 A2 PODc (Ascend 910B1) or Atlas 900 A3 PODc (Ascend 910_9391) | | Host OS | Ubuntu 22.04 | -| CANN | 8.5.1 | +| CANN | 9.0.0 | | Ascend NPU Driver | Installed on host (`npu-smi info` shows devices) | | Docker | >= 20.10 | @@ -32,12 +32,12 @@ Use the pre-built Ascend image that matches your hardware. Official ROLL NPU ima ```bash # For A2 hardware -docker pull roll-registry.cn-hangzhou.cr.aliyuncs.com/roll/pytorch:cann851-910b-py311-torch280-vllm0130 -docker tag roll-registry.cn-hangzhou.cr.aliyuncs.com/roll/pytorch:cann851-910b-py311-torch280-vllm0130 roll:ascend-a2 +docker pull quay.io/ascend/roll:main-a2 +docker tag quay.io/ascend/roll:main-a2 roll:ascend-a2 # For A3 hardware -docker pull roll-registry.cn-hangzhou.cr.aliyuncs.com/roll/pytorch:cann851-a3-py311-torch280-vllm0130 -docker tag roll-registry.cn-hangzhou.cr.aliyuncs.com/roll/pytorch:cann851-a3-py311-torch280-vllm0130 roll:ascend-a3 +docker pull quay.io/ascend/roll:main-a3 +docker tag quay.io/ascend/roll:main-a3 roll:ascend-a3 ``` The current repository includes `docker/Dockerfile.A2` and `docker/Dockerfile.A3` for building custom images. If you maintain a custom image, keep the dependency versions aligned with the pre-built image. @@ -224,41 +224,43 @@ When adapting the GPU RLVR configuration for NPU, the following changes are **re | Item | GPU | NPU | | ---- | --- | --- | -| Training backend | Megatron or DeepSpeed | DeepSpeed only (Megatron not supported) | +| Training backend | Megatron or FSDP2 | FSDP2 only (Megatron not supported on NPU) | | Inference backend | vLLM | vLLM-Ascend | -| Reference model strategy | `megatron_infer` | `hf_infer` | -| Device placement | Colocated mode supported | Colocated mode **not** supported; training and inference must use separate NPUs | +| Reference model strategy | `megatron_infer` | `fsdp2_infer` | | Attention implementation | `flash_attn` or `fa2` | `fa2` via `transformers` (not `flash_attn` package) | | Communication backend | NCCL | HCCL | | Device visibility | `CUDA_VISIBLE_DEVICES` | `ASCEND_RT_VISIBLE_DEVICES` | -| DeepSpeed config | ZeRO-2 or ZeRO-3 | ZeRO-3 + CPU offloading recommended for 7B+ models | +| Sharding config | FSDP2 or Megatron optimizer sharding | FSDP2 with `offload_policy: true` recommended for 7B+ models | ### Complete NPU Configuration Example -Create a YAML config file based on an existing GPU config (such as `examples/qwen2.5-7B-rlvr_megatron/rlvr_config_amd.yaml`). Below is a complete NPU-adapted configuration with key differences marked with `# NPU` comments: +Below is a complete NPU-adapted configuration (adapted from `examples/ascend_examples/qwen3_30b_rlvr_fsdp2.yaml`), with key differences marked with `# NPU` comments: ```yaml -defaults: - - ../config/deepspeed_zero@_here_ - - ../config/deepspeed_zero2@_here_ - - ../config/deepspeed_zero3@_here_ - - ../config/deepspeed_zero3_cpuoffload@_here_ - hydra: run: dir: . output_subdir: null -exp_name: "qwen2.5-7B-rlvr-npu" +exp_name: "qwen3-30BA3B-rlvr-npu" seed: 42 logging_dir: ./output/logs output_dir: ./output +system_envs: + USE_MODELSCOPE: '1' + HCCL_NPU_SOCKET_PORT_RANGE: auto # NPU: allow same-device multi-process HCCL port allocation + VLLM_ASCEND_ENABLE_NZ: '0' # NPU: disable FRACTAL_NZ for RL weight reload flows checkpoint_config: type: file_system - output_dir: /data/models/${exp_name} + output_dir: ./output/models/${exp_name} -num_gpus_per_node: 8 +track_with: tensorboard +tracker_kwargs: + log_dir: ./output/tensorboard/rlvr_npu +rpc_timeout: 72000 + +num_gpus_per_node: 16 max_steps: 500 save_steps: 100 @@ -266,7 +268,7 @@ logging_steps: 1 eval_steps: 10 resume_from_checkpoint: false -rollout_batch_size: 64 +rollout_batch_size: 32 prompt_length: 2048 response_length: 4096 num_return_sequences_in_group: 8 @@ -294,54 +296,53 @@ length_loss_weight: false add_token_level_kl: false whiten_advantages: true -pretrain: Qwen/Qwen2.5-7B -reward_pretrain: Qwen/Qwen2.5-7B - -track_with: tensorboard -tracker_kwargs: - log_dir: ./output/tensorboard/rlvr_npu - -validation: - data_args: - template: qwen2_5 - file_name: - - data/math_benchmarks.jsonl - generating_args: - max_new_tokens: ${response_length} - top_p: 0.6 - top_k: 50 - num_beams: 1 - temperature: 0.6 - num_return_sequences: 1 +pretrain: Qwen/Qwen3-30B-A3B +reward_pretrain: Qwen/Qwen3-30B-A3B actor_train: model_args: - attn_implementation: fa2 # NPU: Use fa2 via transformers, NOT flash_attn disable_gradient_checkpointing: false dtype: bf16 model_type: ~ training_args: learning_rate: 1.0e-6 weight_decay: 0 - per_device_train_batch_size: 1 - gradient_accumulation_steps: 32 + per_device_train_batch_size: 2 + gradient_accumulation_steps: 8 warmup_steps: 20 + num_train_epochs: 50 data_args: template: qwen2_5 file_name: - data/math_deepmath_deal.jsonl - - data/code_KodCode_data.jsonl domain_interleave_probs: - math_rule: 0.5 - code_sandbox: 0.5 + math_rule: 1 dataset_dir: data messages: messages interleave_probs: "1.0" + preprocessing_num_workers: 16 strategy_args: - strategy_name: deepspeed_train # NPU: Must use DeepSpeed, NOT megatron_train - strategy_config: ${deepspeed_zero3_cpuoffload} # NPU: Use ZeRO-3 + CPU offloading for 7B - device_mapping: list(range(0,4)) # NPU: Training on NPUs 0-3 - infer_batch_size: 4 + strategy_name: fsdp2_train # NPU: Must use FSDP2, NOT megatron_train + strategy_config: + fsdp_size: 16 # NPU: FSDP2 sharding size + param_dtype: bf16 + reduce_dtype: bf16 + offload_policy: true # NPU: Enable CPU offloading for large models + apply_expert_patch: true # NPU: Required for MoE models + apply_tiled_mlp: true # NPU: TiledMLP to reduce memory + tiled_num_shards: 8 + reshard_after_forward: true + wrap_policy: # NPU: MoE-specific wrap policy + wrap_embeddings: true + wrap_lm_output: true + moe_experts: + - Qwen3MoeMLP + transformer_layer_cls_to_wrap: + - Qwen3MoeAttention + - Qwen3MoeSparseMoeBlock + use_remove_padding: true + device_mapping: list(range(0,16)) # NPU: Training on NPUs 0-15 + infer_batch_size: 2 actor_infer: model_args: @@ -361,23 +362,32 @@ actor_infer: strategy_config: gpu_memory_utilization: 0.8 block_size: 16 - max_model_len: 8000 - device_mapping: list(range(4,8)) # NPU: Inference on NPUs 4-7 (separate from training) + max_model_len: 6144 + tensor_parallel_size: 2 + enforce_eager: true + load_format: dummy + device_mapping: list(range(0,16)) # NPU: Inference shares NPUs with training infer_batch_size: 1 reference: model_args: - attn_implementation: fa2 # NPU: Use fa2 via transformers disable_gradient_checkpointing: true dtype: bf16 model_type: ~ data_args: template: qwen2_5 strategy_args: - strategy_name: hf_infer # NPU: Use hf_infer, NOT megatron_infer - strategy_config: ~ - device_mapping: list(range(4,8)) # NPU: Share inference NPUs with actor_infer - infer_batch_size: 8 + strategy_name: fsdp2_infer # NPU: Use fsdp2_infer, NOT megatron_infer + strategy_config: + fsdp_size: 16 + param_dtype: bf16 + reduce_dtype: bf16 + apply_tiled_mlp: true + tiled_num_shards: 8 + reshard_after_forward: true + offload_policy: true + device_mapping: list(range(0,16)) # NPU: Reference shares NPUs with training + infer_batch_size: 2 rewards: math_rule: @@ -387,23 +397,13 @@ rewards: data_args: template: qwen2_5 tag_included: [deepmath_103k, aime] - world_size: 4 - infer_batch_size: 1 - code_sandbox: - use_local: true - worker_cls: roll.pipeline.rlvr.rewards.code_sandbox_reward_worker.CodeSandboxRewardWorker - tag_included: [KodCode] - model_args: - model_name_or_path: ${reward_pretrain} - data_args: - template: qwen2_5 - world_size: 4 + world_size: 8 infer_batch_size: 1 ``` ### Key Configuration Changes Explained -#### 1. Training Strategy: DeepSpeed instead of Megatron +#### 1. Training Strategy: FSDP2 instead of Megatron ```yaml # GPU (original) @@ -417,13 +417,18 @@ actor_train: # NPU (adapted) actor_train: strategy_args: - strategy_name: deepspeed_train - strategy_config: ${deepspeed_zero3_cpuoffload} + strategy_name: fsdp2_train + strategy_config: + fsdp_size: 4 + param_dtype: bf16 + reduce_dtype: bf16 + reshard_after_forward: true + offload_policy: true ``` -For 7B models on 4 NPUs, use `deepspeed_zero3_cpuoffload` to avoid OOM. For smaller models (e.g., 0.5B), `deepspeed_zero3` may be sufficient. +For 7B models on 4 NPUs, set `offload_policy: true` to enable CPU offloading and avoid OOM. For smaller models (e.g., 0.5B), `offload_policy: false` may be sufficient. -#### 2. Reference Model: hf_infer instead of megatron_infer +#### 2. Reference Model: fsdp2_infer instead of megatron_infer ```yaml # GPU @@ -434,26 +439,16 @@ reference: # NPU reference: strategy_args: - strategy_name: hf_infer - strategy_config: ~ -``` - -#### 3. Device Mapping: Separate Training and Inference NPUs - -NPU does **not** support colocated mode. Training and inference must run on different NPUs: - -```yaml -actor_train: - device_mapping: list(range(0,4)) # Training: NPUs 0-3 -actor_infer: - device_mapping: list(range(4,8)) # Inference: NPUs 4-7 -reference: - device_mapping: list(range(4,8)) # Shares inference NPUs + strategy_name: fsdp2_infer + strategy_config: + fsdp_size: 4 + param_dtype: bf16 + reduce_dtype: bf16 + reshard_after_forward: true + offload_policy: true ``` -See [Device Mapping Reference](#device-mapping-reference) for more allocation patterns. - -#### 4. Attention Implementation +#### 3. Attention Implementation Use `fa2` through the `transformers` library instead of the `flash_attn` package: @@ -463,7 +458,7 @@ actor_train: attn_implementation: fa2 # NOT flash_attn ``` -#### 5. System Environment Variables +#### 4. System Environment Variables ROLL injects device visibility and Ray runtime variables for workers, but production runs should still set HCCL, memory, vLLM-Ascend, cache, and logging variables explicitly. See the [NPU Environment Configuration Guide](ascend_npu_env_config.md) for the recommended single-node and multi-node settings. @@ -479,7 +474,7 @@ export PYTHONPATH="/workspace/ROLL:$PYTHONPATH" python examples/start_rlvr_pipeline.py \ --config_path ascend_examples \ - --config_name qwen3_8b_rlvr_deepspeed + --config_name qwen3_30b_rlvr_fsdp2 ``` If you save the custom configuration above as `/rlvr_npu.yaml`, use `--config_path --config_name rlvr_npu` instead. @@ -507,10 +502,14 @@ export HCCL_CONNECT_TIMEOUT=3600 export HCCL_EXEC_TIMEOUT=3600 export HCCL_DETERMINISTIC=false export HCCL_OP_EXPANSION_MODE="AIV" +export HCCL_NPU_SOCKET_PORT_RANGE="auto" export HCCL_IF_IP=10.0.0.1 # This node's IP export HCCL_SOCKET_IFNAME="enp194s0f0" # HCCL network interface export HCCL_IF_BASE_PORT=23456 +# vLLM-Ascend RL scenario +export VLLM_ASCEND_ENABLE_NZ=0 + # NPU memory, CPU, vLLM, cache, logging... (same as single-node) # See the NPU Environment Configuration Guide for the full list ``` @@ -530,10 +529,14 @@ export HCCL_CONNECT_TIMEOUT=3600 export HCCL_EXEC_TIMEOUT=3600 export HCCL_DETERMINISTIC=false export HCCL_OP_EXPANSION_MODE="AIV" +export HCCL_NPU_SOCKET_PORT_RANGE="auto" export HCCL_IF_IP=10.0.0.2 # This node's IP export HCCL_SOCKET_IFNAME="enp194s0f0" export HCCL_IF_BASE_PORT=23456 +# vLLM-Ascend RL scenario +export VLLM_ASCEND_ENABLE_NZ=0 + # NPU memory, CPU, vLLM, cache, logging... (same as single-node) ``` @@ -597,8 +600,13 @@ num_gpus_per_node: 8 # Training on Node0 NPUs 0-7 actor_train: strategy_args: - strategy_name: deepspeed_train - strategy_config: ${deepspeed_zero3_cpuoffload} + strategy_name: fsdp2_train + strategy_config: + fsdp_size: 8 + param_dtype: bf16 + reduce_dtype: bf16 + reshard_after_forward: true + offload_policy: true device_mapping: list(range(0,8)) # Inference on Node1 NPUs 0-7 @@ -613,8 +621,13 @@ actor_infer: # Reference model shares inference NPUs reference: strategy_args: - strategy_name: hf_infer - strategy_config: ~ + strategy_name: fsdp2_infer + strategy_config: + fsdp_size: 8 + param_dtype: bf16 + reduce_dtype: bf16 + reshard_after_forward: true + offload_policy: true device_mapping: list(range(8,16)) ``` @@ -690,7 +703,7 @@ save_steps: 100 Set `resume_from_checkpoint` to the checkpoint path to resume training: ```yaml -resume_from_checkpoint: /data/models/qwen2.5-7B-rlvr-npu/checkpoint-100 +resume_from_checkpoint: ./output/models/qwen3-30BA3B-rlvr-npu/checkpoint-100 ``` Or override via the launch command: @@ -699,38 +712,38 @@ Or override via the launch command: python examples/start_rlvr_pipeline.py \ --config_path \ --config_name rlvr_npu \ - resume_from_checkpoint=/data/models/qwen2.5-7B-rlvr-npu/checkpoint-100 + resume_from_checkpoint=./output/models/qwen3-30BA3B-rlvr-npu/checkpoint-100 ``` ## Device Mapping Reference -Since NPU does not support colocated mode, you must allocate separate NPUs for training and inference. Below are common allocation patterns for RLVR: +Below are common NPU allocation patterns for RLVR. You may use colocated mode (training and inference sharing NPUs) or separate NPUs based on your workload and hardware: ### 8-NPU Single Node (7B Model) | Component | NPUs | Count | Notes | | --------- | ---- | ----- | ----- | -| actor_train | 0-3 | 4 | DeepSpeed ZeRO-3 + CPU offloading | +| actor_train | 0-3 | 4 | FSDP2 + CPU offloading | | actor_infer | 4-7 | 4 | vLLM-Ascend | -| reference | 4-7 (shared) | - | hf_infer, shares with actor_infer | +| reference | 4-7 (shared) | - | fsdp2_infer, shares with actor_infer | | reward workers | CPU | - | Math rule & code sandbox run on CPU | ### 16-NPU Single Node (A3, 7B Model) | Component | NPUs | Count | Notes | | --------- | ---- | ----- | ----- | -| actor_train | 0-7 | 8 | DeepSpeed ZeRO-3 | +| actor_train | 0-7 | 8 | FSDP2 | | actor_infer | 8-15 | 8 | vLLM-Ascend | -| reference | 8-15 (shared) | - | hf_infer, shares with actor_infer | +| reference | 8-15 (shared) | - | fsdp2_infer, shares with actor_infer | | reward workers | CPU | - | Math rule & code sandbox run on CPU | ### 2×8-NPU Multi-Node (7B Model) | Component | NPUs | Count | Notes | | --------- | ---- | ----- | ----- | -| actor_train | Node0: 0-7 | 8 | DeepSpeed ZeRO-3 + CPU offloading | +| actor_train | Node0: 0-7 | 8 | FSDP2 + CPU offloading | | actor_infer | Node1: 0-7 | 8 | vLLM-Ascend | -| reference | Node1: 0-7 (shared) | - | hf_infer, shares with actor_infer | +| reference | Node1: 0-7 (shared) | - | fsdp2_infer, shares with actor_infer | | reward workers | CPU | - | Math rule & code sandbox run on CPU | ## Supported Reward Workers on NPU @@ -753,12 +766,11 @@ When using `LLMJudgeRewardWorker`, the judge model requires its own NPU devices Use this checklist when migrating an existing GPU RLVR configuration to NPU: -- [ ] Change `actor_train.strategy_args.strategy_name` from `megatron_train` to `deepspeed_train` -- [ ] Change `actor_train.strategy_args.strategy_config` to `${deepspeed_zero3_cpuoffload}` or `${deepspeed_zero3}` -- [ ] Change `reference.strategy_args.strategy_name` from `megatron_infer` to `hf_infer` -- [ ] Set `reference.strategy_args.strategy_config` to `~` (null) +- [ ] Change `actor_train.strategy_args.strategy_name` from `megatron_train` to `fsdp2_train` +- [ ] Change `actor_train.strategy_args.strategy_config` to FSDP2 config (with `offload_policy: true` for 7B+ models) +- [ ] Change `reference.strategy_args.strategy_name` from `megatron_infer` to `fsdp2_infer` +- [ ] Set `reference.strategy_args.strategy_config` to FSDP2 config matching `actor_train` - [ ] Add `attn_implementation: fa2` to `actor_train.model_args` and `reference.model_args` -- [ ] Ensure `device_mapping` separates training and inference NPUs (no colocated mode) - [ ] Remove any `flash_attn` references - [ ] Remove any Megatron-specific config (e.g., `tensor_model_parallel_size`, `pipeline_model_parallel_size`) - [ ] Verify `llm_judge` reward worker has separate NPU allocation (if used) @@ -776,7 +788,7 @@ The first inference request after model loading triggers operator compilation, w If you encounter OOM with a 7B model on 4 NPUs: -1. Switch to `deepspeed_zero3_cpuoffload` strategy. +1. Switch to `fsdp2_train` strategy with `offload_policy: true`. 2. Reduce `per_device_train_batch_size` to 1. 3. Increase `gradient_accumulation_steps` accordingly. 4. Reduce `max_model_len` in vLLM config (e.g., from 8192 to 4096). @@ -800,7 +812,7 @@ The `triton` package conflicts with `triton-ascend` on NPU. Fix with: ```bash pip uninstall -y triton triton-ascend -pip install triton-ascend==3.2.0 +pip install triton-ascend==3.2.1 --extra-index-url https://mirrors.huaweicloud.com/ascend/repos/pypi ``` For more troubleshooting tips, see the [Ascend NPU FAQ](ascend_npu_faq.md). diff --git a/docs_roll/docs/User Guides/Hardware Support/ascend_usage.md b/docs_roll/docs/User Guides/Hardware Support/ascend_usage.md index b8b6cb136..d529fc677 100644 --- a/docs_roll/docs/User Guides/Hardware Support/ascend_usage.md +++ b/docs_roll/docs/User Guides/Hardware Support/ascend_usage.md @@ -1,21 +1,42 @@ # ROLL x Ascend -Last updated: 11/25/2025. +Last updated: 06/23/2026. We have added support for Huawei Ascend devices in ROLL. -## Hardware Support +## Hardware Compatibility and Supported Operating Systems -Atlas 900 A2 PODc +ROLL's Ascend support is currently validated on training-series Ascend hardware: + +| Product | Support status | Notes | +| ------- | -------------- | ----- | +| Atlas 900 A2 PODc (Ascend 910B1) / Atlas A2 training series | √ | Use `docker/Dockerfile.A2` or the `roll:ascend-a2` image. | +| Atlas 900 A3 PODc (Ascend 910_9391) / Atlas A3 training series | √ | Use `docker/Dockerfile.A3` or the `roll:ascend-a3` image. | +| Atlas A5 training series | √ | Use the A5 installation profile: torch 2.10, vLLM v0.20.2, vLLM-Ascend `main`, and `COMPILE_CUSTOM_KERNELS=1` when building vLLM-Ascend. | +| Atlas A2/A3 inference series and Atlas 200I/500 A2 inference products | x | Current ROLL NPU images and examples target training-series devices. | +| Other Ascend training or inference products | Not validated | Validate the driver, firmware, CANN, `torch_npu`, and vLLM-Ascend versions before use. | + +> In this table, `√` means supported by the current ROLL Ascend Dockerfiles/examples or the manual A5 installation profile, and `x` means not supported in the current ROLL NPU setup. + +Supported operating systems: + +| Deployment scenario | Supported OS | Notes | +| ------------------- | ------------ | ----- | +| Physical host | Ubuntu 22.04 | Recommended and validated by the current ROLL Ascend guides. | +| ROLL Ascend container | Ubuntu 22.04 | The A2/A3 Dockerfiles are based on `quay.io/ascend/cann:9.0.0-*-ubuntu22.04-py3.11`. | +| Atlas A5 manual installation | Ubuntu 22.04 | Use the A5-specific torch/vLLM stack below. Keep the driver, firmware, CANN, and `torch_npu` versions aligned with the target A5 environment. | +| VM/container deployments on other host OS versions | Follow Ascend/CANN compatibility guidance | Check the Ascend compatibility query assistant and the CANN Software Installation OS compatibility notes for the target hardware. | ## Installation ### Basic Environment Setup | Software | Version | -| -------- | ------- | +| -------- |---------| | Python | 3.11 | -| CANN | 8.3.RC1 | +| CANN | 9.0.0 | + +For Atlas A5, keep Python 3.11 and use the A5-specific torch/vLLM stack described in [A5 Installation Profile](#a5-installation-profile). ### Create Conda Environment @@ -31,11 +52,11 @@ conda activate roll To use torch and torch_npu in ROLL, install them using the commands below: ``` -# Use CPU only torch -pip install torch==2.7.1 torchvision==0.22.1 torchaudio==2.7.1 --index-url https://download.pytorch.org/whl/cpu +# Use CPU-only torch when installing outside the pre-built image +pip install torch==2.9.0 torchvision==0.24.0 torchaudio==2.9.0 --index-url https://download.pytorch.org/whl/cpu -# Install torch_npu 2.7.1 -pip install torch_npu==2.7.1 +# Install the torch_npu version matching torch/CANN +pip install torch_npu==2.9.0 ``` ### Install vllm & vllm-ascend @@ -44,7 +65,7 @@ To use vllm in ROLL, compile and install vllm and vllm-ascend as follows: ``` # vllm -git clone -b v0.11.0 --depth 1 https://github.com/vllm-project/vllm.git +git clone -b v0.18.0 --depth 1 https://github.com/vllm-project/vllm.git cd vllm pip install -r requirements/build.txt @@ -52,7 +73,7 @@ VLLM_TARGET_DEVICE=empty pip install -v -e . cd .. # vllm-ascend -git clone -b v0.11.0rc1 --depth 1 https://github.com/vllm-project/vllm-ascend.git +git clone -b v0.18.0 --depth 1 https://github.com/vllm-project/vllm-ascend.git cd vllm-ascend pip install -e . @@ -61,11 +82,37 @@ cd .. Or you could install `vllm` and `vllm-ascend` from pre-built wheel: ``` -# Install vllm-project/vllm. The newest supported version is v0.11.0. -pip install vllm==0.11.0 +# Install vllm-project/vllm. The newest supported version is v0.18.0. +pip install vllm==0.18.0 # Install vllm-project/vllm-ascend from pypi. -pip install vllm-ascend==0.11.0rc1 +pip install vllm-ascend==0.18 +``` + +### A5 Installation Profile + +For Atlas A5, use torch 2.10, vLLM v0.20.2, and vLLM-Ascend from the `main` branch. Set `COMPILE_CUSTOM_KERNELS=1` before installing vLLM-Ascend so its custom kernels are built: + +``` +# Install torch 2.10 +pip install torch==2.10.0 torchvision==0.25.0 torchaudio==2.10.0 --index-url https://download.pytorch.org/whl/cpu + +# Install the torch_npu package matching torch 2.10 and your CANN release +pip install torch_npu==2.10.0 + +# vLLM v0.20.2 +git clone -b v0.20.2 --depth 1 https://github.com/vllm-project/vllm.git +cd vllm +pip install -r requirements/build.txt +VLLM_TARGET_DEVICE=empty pip install -v -e . +cd .. + +# vLLM-Ascend main +git clone -b main --depth 1 https://github.com/vllm-project/vllm-ascend.git +cd vllm-ascend +export COMPILE_CUSTOM_KERNELS=1 +pip install -v -e . +cd .. ``` ### Install ROLL @@ -74,6 +121,7 @@ pip install vllm-ascend==0.11.0rc1 git clone https://github.com/alibaba/ROLL.git cd ROLL pip install -r requirements_common.txt +pip install -e . cd .. ``` @@ -81,22 +129,23 @@ cd .. | Software | Description | | --------------------------- | ------------- | -| transformers | >= v4.57.1 | +| transformers | >= v4.57.6 | | flash_attn | not supported | | transformer-engine[pytorch] | not supported | -1. `transformers` v4.57.1 supports enabling `--flash_attention_2`. +1. `transformers` v4.57.6 supports enabling `--flash_attention_2`. 2. `flash_attn` acceleration is not supported currently. 3. `transformer-engine[pytorch]` is currently not supported. ``` -pip install transformers==4.57.1 +pip install transformers==4.57.6 ``` ## Quick Start: Single-Node Deployment Before full usage, we recommend testing the single-node pipeline to verify your environment and installation. -Since Megatron-LM training is not yet supported, first change `strategy_args` in the relevant files to use the `fsdp2` option. +Since Megatron-LM is not supported on NPU, first change `strategy_args` in the relevant files to use the `fsdp2` option. + 1. Run the single-node pipeline via shell: @@ -121,11 +170,10 @@ python examples/start_agentic_pipeline.py \ | Feature | Example | Training Backend | Inference Backend | Hardware | | --------------- | ------------------------------------------------------------ | ---------------- | ----------------- | ----------------- | -| Agentic | examples/qwen2.5-0.5B-agentic/run_agentic_pipeline_sokoban.sh | FSDP2 | vLLM | Atlas 900 A2 PODc | -| Agentic-Rollout | examples/qwen2.5-0.5B-agentic/run_agentic_rollout_sokoban.sh | FSDP2 | vLLM | Atlas 900 A2 PODc | -| DPO | examples/qwen2.5-3B-dpo_megatron/run_dpo_pipeline.sh | FSDP2 | vLLM | Atlas 900 A2 PODc | -| RLVR | examples/qwen2.5-7B-rlvr_megatron/run_rlvr_pipeline.sh | FSDP2 | vLLM | Atlas 900 A2 PODc | +| Agentic | examples/qwen2.5-0.5B-agentic/run_agentic_pipeline_sokoban.sh | FSDP2 | vLLM | Atlas 900 A2/A3 PODc | +| Agentic-Rollout | examples/qwen2.5-0.5B-agentic/run_agentic_rollout_sokoban.sh | FSDP2 | vLLM | Atlas 900 A2/A3 PODc | +| RLVR | examples/ascend_examples/run_rlvr_pipeline.sh | FSDP2 | vLLM | Atlas 900 A2/A3/A5 training series | ## Disclaimer -The Ascend support provided in ROLL is intended as a reference example. For production use, please consult official channels. \ No newline at end of file +The Ascend support provided in ROLL is intended as a reference example. For production use, please consult official channels. diff --git a/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_docker_usage.md b/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_docker_usage.md index 940d804a3..6a7dc97d5 100644 --- a/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_docker_usage.md +++ b/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_docker_usage.md @@ -1,8 +1,8 @@ # 使用 Docker 在昇腾 NPU 上运行 ROLL -最后更新：2026/04/27。 +最后更新：2026/06/23。 -本指南介绍如何使用 `Dockerfile.A2` 和 `Dockerfile.A3` 在**华为昇腾 NPU** 上构建并运行 ROLL。 +本指南介绍如何在**华为昇腾 NPU** 上获取、构建并运行 ROLL 镜像。推荐优先使用预构建镜像；如需自定义依赖，再使用 `Dockerfile.A2` 或 `Dockerfile.A3` 构建。Atlas A5 当前使用 [ROLL x Ascend](ascend_usage.md) 中的手动安装配置。 ## 硬件与软件要求 @@ -10,32 +10,64 @@ | ---- | ------------- | ------------- | | 硬件 | Atlas 900 A2 PODc（Ascend 910B1） | Atlas 900 A3 PODc（Ascend 910_9391） | | 宿主机操作系统 | Ubuntu 22.04 | Ubuntu 22.04 | -| CANN | 8.5.1 | 8.5.1 | +| CANN | 9.0.0 | 9.0.0 | | Python | 3.11 | 3.11 | | Docker | >= 20.10 | >= 20.10 | | 昇腾 NPU 驱动 | 已安装在宿主机上 | 已安装在宿主机上 | +本 Docker 指南覆盖 A2/A3 Dockerfile。Atlas A5 请使用手动安装配置：torch 2.10、vLLM v0.20.2、vLLM-Ascend `main`，并在构建 vLLM-Ascend 时设置 `COMPILE_CUSTOM_KERNELS=1`。 + ## 主要组件 两个 Dockerfile 安装的核心依赖版本相同： | 组件 | 版本 | | ---- | ---- | -| PyTorch | 2.8.0+cpu | -| vLLM | 0.13.0 | -| vLLM-Ascend | 0.13.0 | -| DeepSpeed | 0.16.4 | +| PyTorch | 2.9.0+cpu | +| vLLM | 0.18.0 | +| vLLM-Ascend | 0.18 | | Transformers | 4.57.6 | -| triton-ascend | 3.2.0 | +| triton-ascend | 3.2.1 | + +Atlas A5 使用更新的手动安装版本组合： + +| 组件 | Atlas A5 版本 / 设置 | +| ---- | -------------------- | +| PyTorch | 2.10 | +| vLLM | v0.20.2 | +| vLLM-Ascend | `main` 分支 | +| 必需构建变量 | `COMPILE_CUSTOM_KERNELS=1` | 主要区别在于基础镜像和 SOC 版本： | 项目 | Dockerfile.A2 | Dockerfile.A3 | | ---- | ------------- | ------------- | -| 基础镜像 | `quay.io/ascend/cann:8.5.1-910b-ubuntu22.04-py3.11` | `quay.io/ascend/cann:8.5.1-a3-ubuntu22.04-py3.11` | +| 基础镜像 | `quay.io/ascend/cann:9.0.0-910b-ubuntu22.04-py3.11` | `quay.io/ascend/cann:9.0.0-a3-ubuntu22.04-py3.11` | | SOC_VERSION | `ascend910b1` | `ascend910_9391` | -## 构建 Docker 镜像 +## 获取 Docker 镜像 + +### 方式 A：使用预构建镜像（推荐） + +根据你的硬件拉取对应镜像，并打成本指南后续命令使用的本地标签： + +**Atlas 900 A2 PODc（Ascend 910B1）：** + +```bash +docker pull quay.io/ascend/roll:main-a2 +docker tag quay.io/ascend/roll:main-a2 roll:ascend-a2 +``` + +**Atlas 900 A3 PODc（Ascend 910_9391）：** + +```bash +docker pull quay.io/ascend/roll:main-a3 +docker tag quay.io/ascend/roll:main-a3 roll:ascend-a3 +``` + +可用镜像标签请以 https://quay.io/repository/ascend/roll?tab=tags 为准。如果你使用预构建镜像，可以直接跳到 [运行容器](#运行容器)。 + +### 方式 B：从 Dockerfile 构建镜像 ### 1. 克隆 ROLL 仓库 @@ -88,6 +120,7 @@ docker run -dit \ --device /dev/davinci4 \ --device /dev/davinci5 \ --device /dev/davinci6 \ + --device /dev/davinci7 \ --device /dev/davinci_manager \ --device /dev/devmm_svm \ --device /dev/hisi_hdc \ @@ -115,6 +148,7 @@ docker run -dit \ --device /dev/davinci4 \ --device /dev/davinci5 \ --device /dev/davinci6 \ + --device /dev/davinci7 \ --device /dev/davinci_manager \ --device /dev/devmm_svm \ --device /dev/hisi_hdc \ @@ -190,7 +224,7 @@ npu-smi info env | grep -E "ASCEND|LD_LIBRARY_PATH|PATH" # 验证 Python 包 -python -c "import torch; import torch_npu; print(torch_npu.npu.is_available())" +python -c "import torch; import torch_npu; print(torch.npu.is_available())" python -c "import vllm; print(f'vllm: {vllm.__version__}')" python -c "import vllm_ascend; print(f'vllm_ascend available')" ``` @@ -199,21 +233,20 @@ python -c "import vllm_ascend; print(f'vllm_ascend available')" ### 重要配置说明 -由于昇腾 NPU 上暂不支持 Megatron-LM 训练，需要使用 **DeepSpeed** 作为训练后端。请确保配置文件中使用以下设置： +由于昇腾 NPU 上不支持 Megatron-LM 训练，需要使用 **FSDP2** 作为训练后端。请确保配置文件中使用以下设置： -1. 将 `strategy_args` 设置为使用 DeepSpeed -2. 设置 `device_mapping`，确保训练和推理在不同的 NPU 卡上执行 +1. 将 `strategy_args` 设置为使用 FSDP2 ### 示例：RLVR 流水线 ```bash python examples/start_rlvr_pipeline.py \ - --config_path qwen2.5-7B-rlvr_megatron \ - --config_name rlvr_config_amd + --config_path ascend_examples \ + --config_name qwen3_30b_rlvr_fsdp2 ``` -> **注意：** `rlvr_config_amd` 配置专为非 NVIDIA 硬件设计，使用 DeepSpeed 作为训练后端。请根据你的 NPU 拓扑调整配置文件中的 `device_mapping`。 +> **注意：** `qwen3_30b_rlvr_fsdp2` 配置专为昇腾 NPU 设计，使用 FSDP2 作为训练后端。请根据你的 NPU 拓扑调整配置文件中的 `device_mapping`。 ## 常见问题 diff --git a/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_npu_env_config.md b/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_npu_env_config.md index 9a1a5dbd9..a41b892c2 100644 --- a/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_npu_env_config.md +++ b/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_npu_env_config.md @@ -1,6 +1,6 @@ # 昇腾 NPU 环境变量配置指南 -最后更新：2026/04/27。 +最后更新：2026/06/23。 本文档说明在华为昇腾 NPU 上运行 ROLL 时涉及的关键环境变量，涵盖设备管理、HCCL 通信、显存优化、CPU 调度、vLLM-Ascend 推理及调试日志等方面。 @@ -79,6 +79,7 @@ export MASTER_PORT=6379 | `HCCL_DETERMINISTIC` | `false` | 关闭确定性计算，开启会显著降低通信性能 | | `HCCL_OP_EXPANSION_MODE` | `"AIV"` | 通信算法展开位置，`AIV` 使用 Vector Core，性能优于 `AI_CPU`/`HOST`/`HOST_TS` | | `HCCL_BUFFSIZE` | 如 `"2147483648"` | HCCL 通信缓冲区大小（字节），大数据量场景可增大 | +| `HCCL_NPU_SOCKET_PORT_RANGE` | `auto` | 同一 NPU 上运行多个 worker 进程时，允许 HCCL 自动分配非默认的 device 侧网卡端口 | | `HCCL_IF_IP` | 节点 IP 地址 | 指定 HCCL 跨节点通信使用的 IP 地址，多机训练必需 | | `HCCL_SOCKET_IFNAME` | 如 `"enp194s0f0"` | HCCL Socket 通信使用的网卡名称，所有节点必须一致 | | `HCCL_IF_BASE_PORT` | 如 `23456` | HCCL 跨节点通信基础端口，确保端口未被防火墙拦截 | @@ -90,6 +91,7 @@ export MASTER_PORT=6379 export HCCL_CONNECT_TIMEOUT=3600 export HCCL_DETERMINISTIC=false export HCCL_OP_EXPANSION_MODE="AIV" +export HCCL_NPU_SOCKET_PORT_RANGE="auto" ``` 示例（多机）： @@ -99,6 +101,7 @@ export HCCL_CONNECT_TIMEOUT=3600 export HCCL_EXEC_TIMEOUT=3600 export HCCL_DETERMINISTIC=false export HCCL_OP_EXPANSION_MODE="AIV" +export HCCL_NPU_SOCKET_PORT_RANGE="auto" export HCCL_IF_IP=$(hostname -I | awk '{print $1}') export HCCL_SOCKET_IFNAME="enp194s0f0" export HCCL_IF_BASE_PORT=23456 @@ -151,8 +154,7 @@ export CPU_AFFINITY_CONF=1,npu0:0-1,npu1:2-3,npu2:4-5,npu3:6-7 | `VLLM_USE_V1` | `1` | 启用 vLLM V1 架构，vLLM-Ascend 必需 | | `VLLM_ATTENTION_BACKEND` | `XFORMERS` | vLLM 注意力计算后端 | | `VLLM_ASCEND_ENABLE_FLASHCOMM` | `1` | 启用昇腾 FlashComm 高速通信优化 | -| `VLLM_ASCEND_ENABLE_DENSE_OPTIMIZE` | `1` | 启用大模型稠密计算优化 | -| `VLLM_ASCEND_ENABLE_PREFETCH_MLP` | `1` | 启用 MLP 层权重预取 | +| `VLLM_ASCEND_ENABLE_PREFETCH_MLP` | `1` | 启用 MLP 层权重预取。它替代了较早版本中的 dense optimize 开关。 | | `VLLM_ASCEND_ENABLE_TOPK_OPTIMIZE` | `1` | 启用 TopK 算子融合优化，提升生成解码性能 | | `VLLM_ASCEND_MODEL_EXECUTE_TIME_OBSERVE` | `1` | 打印 prefill/decode 阶段耗时详情（调试用） | | `VLLM_ASCEND_TRACE_RECOMPILES` | `1` | 追踪算子重编译，用于调试性能问题 | @@ -164,10 +166,26 @@ export CPU_AFFINITY_CONF=1,npu0:0-1,npu1:2-3,npu2:4-5,npu3:6-7 export VLLM_USE_V1=1 export VLLM_ATTENTION_BACKEND=XFORMERS export VLLM_ASCEND_ENABLE_FLASHCOMM=1 -export VLLM_ASCEND_ENABLE_DENSE_OPTIMIZE=1 export VLLM_ASCEND_ENABLE_PREFETCH_MLP=1 ``` +## vLLM-Ascend 构建相关变量 + +以下变量用于从源码构建 vLLM-Ascend。请在执行 `pip install -e .` 前设置；它不需要在每次运行 ROLL 时都导出。 + +| 变量 | 推荐值 | 说明 | +| ---- | ------ | ---- | +| `COMPILE_CUSTOM_KERNELS` | Atlas A5 设置为 `1` | 编译 vLLM-Ascend 自定义 kernel。使用 vLLM-Ascend `main` 的 Atlas A5 安装配置需要设置该变量。 | + +示例（Atlas A5）： + +```bash +git clone -b main --depth 1 https://github.com/vllm-project/vllm-ascend.git +cd vllm-ascend +export COMPILE_CUSTOM_KERNELS=1 +pip install -v -e . +``` + ## CANN 日志与调试变量 | 变量 | 推荐值 | 说明 | @@ -243,7 +261,6 @@ export OMP_NUM_THREADS=1 # vLLM-Ascend 推理 export VLLM_USE_V1=1 export VLLM_ASCEND_ENABLE_FLASHCOMM=1 -export VLLM_ASCEND_ENABLE_DENSE_OPTIMIZE=1 export VLLM_ASCEND_ENABLE_PREFETCH_MLP=1 # 算子编译缓存 diff --git a/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_npu_examples.md b/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_npu_examples.md index e77c1531e..a04295c94 100644 --- a/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_npu_examples.md +++ b/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_npu_examples.md @@ -8,11 +8,11 @@ 运行本样例前，请确保： -1. 已拉取与硬件匹配的预构建昇腾镜像（参见 [Docker 使用指南](ascend_docker_usage.md)）。 +1. 已拉取或构建与硬件匹配的昇腾镜像（参见 [Docker 使用指南](ascend_docker_usage.md)）。 2. 已在容器内验证环境（参见 [验证环境](ascend_docker_usage.md#验证环境)）。 3. 已将模型权重下载到容器可访问的目录。 -当前仓库在 `examples/ascend_examples` 中提供可直接运行的昇腾 RLVR 示例，包括 `qwen3_8b_rlvr_deepspeed.yaml` 和 `run_rlvr_pipeline.sh`。 +当前仓库在 `examples/ascend_examples` 中提供可直接运行的昇腾 RLVR 示例，包括 `qwen3_30b_rlvr_fsdp2.yaml` 和 `run_rlvr_pipeline.sh`。 ## GPU 与 NPU 的关键差异 @@ -21,15 +21,14 @@ | 项目 | GPU | NPU | | ---- | --- | --- | -| 训练后端 | Megatron 或 DeepSpeed | 仅 DeepSpeed（不支持 Megatron） | -| 设备放置 | 支持 Colocated 模式 | **不支持** Colocated 模式；训练和推理必须使用不同的 NPU 卡 | +| 训练后端 | Megatron 或 FSDP2 | 仅 FSDP2（NPU 不支持 Megatron） | | 注意力实现 | `flash_attn` 或 `fa2` | 通过 `transformers` 使用 `fa2`（不能使用 `flash_attn` 包） | | 通信后端 | NCCL | HCCL | | 设备可见性 | `CUDA_VISIBLE_DEVICES` | `ASCEND_RT_VISIBLE_DEVICES` | ## 样例 1：单机 Agentic 流水线（Qwen2.5-0.5B） -本样例在单个 8 卡 NPU 节点上使用 DeepSpeed ZeRO-3 运行 FrozenLake Agentic 流水线。 +本样例在单个 8 卡 NPU 节点上使用 FSDP2 运行 FrozenLake Agentic 流水线。 ### 步骤 1：启动容器 @@ -68,6 +67,7 @@ docker run -dit \ export HCCL_CONNECT_TIMEOUT=3600 export HCCL_DETERMINISTIC=false export HCCL_OP_EXPANSION_MODE="AIV" +export HCCL_NPU_SOCKET_PORT_RANGE="auto" # NPU 显存 export NPU_MEMORY_FRACTION=0.96 @@ -82,8 +82,8 @@ export OMP_NUM_THREADS=1 # vLLM-Ascend 推理 export VLLM_USE_V1=1 +export VLLM_ASCEND_ENABLE_NZ=0 export VLLM_ASCEND_ENABLE_FLASHCOMM=1 -export VLLM_ASCEND_ENABLE_DENSE_OPTIMIZE=1 export VLLM_ASCEND_ENABLE_PREFETCH_MLP=1 # 算子编译缓存 @@ -99,15 +99,11 @@ export ATB_LOG_LEVEL=ERROR ### 步骤 3：创建 NPU 配置文件 -创建 YAML 配置文件（如 `agentic_frozen_lake_npu.yaml`），以下为 NPU 专用配置。与 GPU 配置的关键差异以 `# NPU` 注释标记： +创建 YAML 配置文件 `examples/agentic_frozen_lake_npu/agentic_frozen_lake_npu.yaml`，以下为 NPU 专用配置。该路径很重要：配置中的 `defaults: - ../config/traj_envs@_here_` 依赖 `examples/` 下的相对目录结构；如果放到其他目录，需要同步调整 `defaults` 路径。与 GPU 配置的关键差异以 `# NPU` 注释标记： ```yaml defaults: - ../config/traj_envs@_here_ - - ../config/deepspeed_zero@_here_ - - ../config/deepspeed_zero2@_here_ - - ../config/deepspeed_zero3@_here_ - - ../config/deepspeed_zero3_cpuoffload@_here_ hydra: run: @@ -124,6 +120,8 @@ system_envs: HCCL_CONNECT_TIMEOUT: "3600" HCCL_DETERMINISTIC: "false" HCCL_OP_EXPANSION_MODE: "AIV" + HCCL_NPU_SOCKET_PORT_RANGE: "auto" + VLLM_ASCEND_ENABLE_NZ: "0" NPU_MEMORY_FRACTION: "0.96" CPU_AFFINITY_CONF: "2" OMP_NUM_THREADS: "1" @@ -172,8 +170,13 @@ actor_train: data_args: template: qwen2_5 strategy_args: - strategy_name: deepspeed_train # NPU: 必须使用 DeepSpeed，不能用 megatron_train - strategy_config: ${deepspeed_zero3} + strategy_name: fsdp2_train + strategy_config: + fsdp_size: 4 + param_dtype: bf16 + reduce_dtype: bf16 + reshard_after_forward: true + offload_policy: false # NPU: 必须使用 FSDP2，不能用 megatron_train device_mapping: list(range(0,4)) # NPU: 训练使用 NPU 0-3 infer_batch_size: 2 @@ -208,8 +211,13 @@ reference: data_args: template: qwen2_5 strategy_args: - strategy_name: hf_infer - strategy_config: ~ + strategy_name: fsdp2_infer + strategy_config: + fsdp_size: 4 + param_dtype: bf16 + reduce_dtype: bf16 + reshard_after_forward: true + offload_policy: false device_mapping: list(range(4,8)) # NPU: 与 actor_infer 共享推理卡 infer_batch_size: 2 @@ -253,13 +261,13 @@ cd /workspace/ROLL export PYTHONPATH="/workspace/ROLL:$PYTHONPATH" python examples/start_agentic_pipeline.py \ - --config_path \ + --config_path agentic_frozen_lake_npu \ --config_name agentic_frozen_lake_npu ``` -## 样例 2：单机 RLVR 流水线（Qwen3-8B） +## 样例 2：单机 RLVR 流水线（Qwen3-30B-A3B） -本样例使用仓库中的 `examples/ascend_examples/qwen3_8b_rlvr_deepspeed.yaml` 配置在昇腾 NPU 上运行 RLVR 流水线。 +本样例使用仓库中的 `examples/ascend_examples/qwen3_30b_rlvr_fsdp2.yaml` 配置在昇腾 NPU 上运行 RLVR 流水线。 ### 关键配置 @@ -269,6 +277,8 @@ system_envs: HCCL_CONNECT_TIMEOUT: "3600" HCCL_DETERMINISTIC: "false" HCCL_OP_EXPANSION_MODE: "AIV" + HCCL_NPU_SOCKET_PORT_RANGE: "auto" + VLLM_ASCEND_ENABLE_NZ: "0" NPU_MEMORY_FRACTION: "0.96" CPU_AFFINITY_CONF: "2" OMP_NUM_THREADS: "1" @@ -277,26 +287,25 @@ system_envs: rollout_batch_size: 32 prompt_length: 2048 -response_length: 8192 +response_length: 4096 num_return_sequences_in_group: 8 -pretrain: Qwen/Qwen3-8B-Base -reward_pretrain: Qwen/Qwen3-8B-Base +pretrain: Qwen/Qwen3-30B-A3B +reward_pretrain: Qwen/Qwen3-30B-A3B actor_train: model_args: - attn_implementation: fa2 # NPU: 通过 transformers 使用 fa2，不能使用 flash_attn disable_gradient_checkpointing: false dtype: bf16 model_type: ~ training_args: learning_rate: 1.0e-6 weight_decay: 0 - per_device_train_batch_size: 1 - gradient_accumulation_steps: 32 + per_device_train_batch_size: 2 + gradient_accumulation_steps: 8 warmup_steps: 20 data_args: - template: qwen3 + template: qwen2_5 file_name: - data/math_deepmath_deal.jsonl domain_interleave_probs: @@ -304,10 +313,28 @@ actor_train: dataset_dir: data messages: messages interleave_probs: "1.0" + preprocessing_num_workers: 16 strategy_args: - strategy_name: deepspeed_train # NPU: 必须使用 DeepSpeed - strategy_config: ${deepspeed_zero3} - device_mapping: list(range(0,8)) # NPU: 训练使用 NPU 0-7 + strategy_name: fsdp2_train + strategy_config: + fsdp_size: 16 + param_dtype: bf16 + reduce_dtype: bf16 + offload_policy: true + apply_expert_patch: true # NPU: MoE 模型必须启用 + apply_tiled_mlp: true # NPU: TiledMLP 降低显存 + tiled_num_shards: 8 + reshard_after_forward: true + wrap_policy: # NPU: MoE 专用 wrap policy + wrap_embeddings: true + wrap_lm_output: true + moe_experts: + - Qwen3MoeMLP + transformer_layer_cls_to_wrap: + - Qwen3MoeAttention + - Qwen3MoeSparseMoeBlock + use_remove_padding: true + device_mapping: list(range(0,16)) # NPU: 训练使用 NPU 0-15 infer_batch_size: 2 actor_infer: @@ -322,15 +349,18 @@ actor_infer: temperature: 0.99 num_return_sequences: ${num_return_sequences_in_group} data_args: - template: qwen3 + template: qwen2_5 strategy_args: strategy_name: vllm strategy_config: gpu_memory_utilization: 0.8 block_size: 16 - max_model_len: 8000 - device_mapping: list(range(8,12)) # NPU: 推理使用 NPU 8-11 - infer_batch_size: 4 + max_model_len: 6144 + tensor_parallel_size: 2 + enforce_eager: true + load_format: dummy + device_mapping: list(range(0,16)) # NPU: 推理与训练共享 NPU + infer_batch_size: 1 reference: model_args: @@ -338,12 +368,19 @@ reference: dtype: bf16 model_type: ~ data_args: - template: qwen3 + template: qwen2_5 strategy_args: - strategy_name: hf_infer - strategy_config: ~ - device_mapping: list(range(12,16)) # NPU: Reference 使用 NPU 12-15 - infer_batch_size: 1 + strategy_name: fsdp2_infer + strategy_config: + fsdp_size: 16 + param_dtype: bf16 + reduce_dtype: bf16 + apply_tiled_mlp: true + tiled_num_shards: 8 + reshard_after_forward: true + offload_policy: true + device_mapping: list(range(0,16)) # NPU: Reference 与训练共享 NPU + infer_batch_size: 2 rewards: math_rule: @@ -351,8 +388,8 @@ rewards: model_args: model_name_or_path: ${reward_pretrain} data_args: - template: qwen3 - tag_included: [deepmath_103k, MATH-500, OlympiadBench, minervamath, aime2025, gsm8k, aime, amc23, math_rule] + template: qwen2_5 + tag_included: [deepmath_103k, aime] world_size: 8 infer_batch_size: 1 ``` @@ -365,7 +402,7 @@ export PYTHONPATH="/workspace/ROLL:$PYTHONPATH" python examples/start_rlvr_pipeline.py \ --config_path ascend_examples \ - --config_name qwen3_8b_rlvr_deepspeed + --config_name qwen3_30b_rlvr_fsdp2 ``` ## 样例 3：多机分布式训练 @@ -418,7 +455,7 @@ python examples/start_rlvr_pipeline.py \ ip addr # 或用 NPU 工具查看 HCCL 接口 -for i in {0..7}; do hccn_tool -i $i -ip -g; done +for i in $(seq 0 7); do hccn_tool -i $i -ip -g; done # NPU 设备 IP 通常在高速互联网络上（如 192.168.x.x）。 # 使用对应的以太网接口名称（如 enp194s0f0, eth0）作为 HCCL_SOCKET_IFNAME。 @@ -463,16 +500,16 @@ docker run -dit \ ```bash # 检查链路状态（全部应显示 "up"） -for i in {0..7}; do hccn_tool -i $i -link -g; done +for i in $(seq 0 7); do hccn_tool -i $i -link -g; done # 检查 TLS 一致性（所有卡应显示相同的 switch 值） -for i in {0..7}; do hccn_tool -i $i -tls -g; done | grep switch +for i in $(seq 0 7); do hccn_tool -i $i -tls -g; done | grep switch # 若 TLS 不一致，在所有节点的所有卡上统一关闭： -for i in {0..7}; do hccn_tool -i $i -tls -s enable 0; done +for i in $(seq 0 7); do hccn_tool -i $i -tls -s enable 0; done # 查看 NPU 设备 IP -for i in {0..7}; do hccn_tool -i $i -ip -g; done +for i in $(seq 0 7); do hccn_tool -i $i -ip -g; done # 测试跨节点连通性（在节点 B 上执行，替换为节点 A 的 device IP） hccn_tool -i 0 -ping -g address <节点A的device_ip> @@ -495,6 +532,7 @@ export HCCL_CONNECT_TIMEOUT=3600 export HCCL_EXEC_TIMEOUT=3600 export HCCL_DETERMINISTIC=false export HCCL_OP_EXPANSION_MODE="AIV" +export HCCL_NPU_SOCKET_PORT_RANGE="auto" export HCCL_IF_IP= # 当前节点 IP export HCCL_SOCKET_IFNAME=<网卡名称> # 例如 enp194s0f0 export HCCL_IF_BASE_PORT=23456 @@ -512,8 +550,8 @@ export OMP_NUM_THREADS=1 # === vLLM-Ascend 推理 === export VLLM_USE_V1=1 +export VLLM_ASCEND_ENABLE_NZ=0 export VLLM_ASCEND_ENABLE_FLASHCOMM=1 -export VLLM_ASCEND_ENABLE_DENSE_OPTIMIZE=1 export VLLM_ASCEND_ENABLE_PREFETCH_MLP=1 # === 算子编译缓存 === @@ -620,8 +658,13 @@ num_gpus_per_node: 8 # 训练在节点0的 NPU 0-7 actor_train: strategy_args: - strategy_name: deepspeed_train - strategy_config: ${deepspeed_zero3_cpuoffload} + strategy_name: fsdp2_train + strategy_config: + fsdp_size: 8 + param_dtype: bf16 + reduce_dtype: bf16 + reshard_after_forward: true + offload_policy: true device_mapping: list(range(0,8)) # 推理在节点1的 NPU 0-7 @@ -637,28 +680,32 @@ actor_infer: # Reference 模型共享推理卡 reference: strategy_args: - strategy_name: hf_infer - strategy_config: ~ + strategy_name: fsdp2_infer + strategy_config: + fsdp_size: 8 + param_dtype: bf16 + reduce_dtype: bf16 + reshard_after_forward: true + offload_policy: true device_mapping: list(range(8,16)) ``` 完整的多机 RLVR 配置示例（2 节点 × 8 卡）： ```yaml -defaults: - - ../config/deepspeed_zero@_here_ - - ../config/deepspeed_zero3@_here_ - - ../config/deepspeed_zero3_cpuoffload@_here_ - hydra: run: dir: . output_subdir: null -exp_name: "qwen2.5-7B-rlvr-npu-multinode" +exp_name: "qwen3-30BA3B-rlvr-npu-multinode" seed: 42 logging_dir: /data/logs output_dir: /data/output +system_envs: + USE_MODELSCOPE: '1' + HCCL_NPU_SOCKET_PORT_RANGE: "auto" + VLLM_ASCEND_ENABLE_NZ: "0" checkpoint_config: type: file_system @@ -672,7 +719,7 @@ logging_steps: 1 eval_steps: 10 resume_from_checkpoint: false -rollout_batch_size: 64 +rollout_batch_size: 32 prompt_length: 2048 response_length: 4096 num_return_sequences_in_group: 8 @@ -681,37 +728,52 @@ ppo_epochs: 1 adv_estimator: "reinforce" whiten_advantages: true -pretrain: /data/models/Qwen2.5-7B -reward_pretrain: /data/models/Qwen2.5-7B +pretrain: Qwen/Qwen3-30B-A3B +reward_pretrain: Qwen/Qwen3-30B-A3B actor_train: model_args: - attn_implementation: fa2 disable_gradient_checkpointing: false dtype: bf16 model_type: ~ training_args: learning_rate: 1.0e-6 weight_decay: 0 - per_device_train_batch_size: 1 - gradient_accumulation_steps: 32 + per_device_train_batch_size: 2 + gradient_accumulation_steps: 8 warmup_steps: 20 data_args: template: qwen2_5 file_name: - data/math_deepmath_deal.jsonl - - data/code_KodCode_data.jsonl domain_interleave_probs: - math_rule: 0.5 - code_sandbox: 0.5 + math_rule: 1 dataset_dir: /data/datasets messages: messages interleave_probs: "1.0" + preprocessing_num_workers: 16 strategy_args: - strategy_name: deepspeed_train - strategy_config: ${deepspeed_zero3_cpuoffload} + strategy_name: fsdp2_train + strategy_config: + fsdp_size: 8 + param_dtype: bf16 + reduce_dtype: bf16 + offload_policy: true + apply_expert_patch: true + apply_tiled_mlp: true + tiled_num_shards: 8 + reshard_after_forward: true + wrap_policy: + wrap_embeddings: true + wrap_lm_output: true + moe_experts: + - Qwen3MoeMLP + transformer_layer_cls_to_wrap: + - Qwen3MoeAttention + - Qwen3MoeSparseMoeBlock + use_remove_padding: true device_mapping: list(range(0,8)) # 节点0 NPU 0-7 用于训练 - infer_batch_size: 4 + infer_batch_size: 2 actor_infer: model_args: @@ -731,23 +793,32 @@ actor_infer: strategy_config: gpu_memory_utilization: 0.8 block_size: 16 - max_model_len: 8000 + max_model_len: 6144 + tensor_parallel_size: 2 + enforce_eager: true + load_format: dummy device_mapping: list(range(8,16)) # 节点1 NPU 0-7 用于推理 infer_batch_size: 1 reference: model_args: - attn_implementation: fa2 disable_gradient_checkpointing: true dtype: bf16 model_type: ~ data_args: template: qwen2_5 strategy_args: - strategy_name: hf_infer - strategy_config: ~ + strategy_name: fsdp2_infer + strategy_config: + fsdp_size: 8 + param_dtype: bf16 + reduce_dtype: bf16 + apply_tiled_mlp: true + tiled_num_shards: 8 + reshard_after_forward: true + offload_policy: true device_mapping: list(range(8,16)) # 共享推理卡 - infer_batch_size: 8 + infer_batch_size: 2 rewards: math_rule: @@ -757,17 +828,7 @@ rewards: data_args: template: qwen2_5 tag_included: [deepmath_103k, aime] - world_size: 4 - infer_batch_size: 1 - code_sandbox: - use_local: true - worker_cls: roll.pipeline.rlvr.rewards.code_sandbox_reward_worker.CodeSandboxRewardWorker - tag_included: [KodCode] - model_args: - model_name_or_path: ${reward_pretrain} - data_args: - template: qwen2_5 - world_size: 4 + world_size: 8 infer_batch_size: 1 ``` @@ -798,7 +859,7 @@ rewards: ## 设备映射参考 -由于 NPU 不支持 colocated 模式，必须为训练和推理分配不同的 NPU 卡。以下是常见的分配方案： +以下是常见的 NPU 分配方案。可以根据工作负载和硬件情况选择共卡模式（训练和推理共享 NPU）或分离模式： ### 8 卡单机 @@ -837,7 +898,7 @@ rewards: 如果在 4 张 NPU 上运行 7B 模型遇到 OOM： -1. 切换到 `deepspeed_zero3_cpuoffload` 策略。 +1. 切换到 `fsdp2_train` 策略并设置 `offload_policy: true`。 2. 将 `per_device_train_batch_size` 减小到 1。 3. 相应增大 `gradient_accumulation_steps`。 4. 减小 vLLM 配置中的 `max_model_len`（如从 8192 减到 4096）。 diff --git a/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_npu_faq.md b/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_npu_faq.md index adc9ea8cf..fdb3e427c 100644 --- a/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_npu_faq.md +++ b/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_npu_faq.md @@ -88,33 +88,20 @@ source /usr/local/Ascend/nnal/atb/set_env.sh ```bash pip uninstall -y triton triton-ascend -pip install triton-ascend==3.2.0 +pip install triton-ascend==3.2.1 --extra-index-url https://mirrors.huaweicloud.com/ascend/repos/pypi ``` ## 训练配置 -### 不支持 Colocated 模式 - -**现象：** `actor_train` 和 `actor_infer` 共用同一组 NPU 设备时训练失败。 - -**解决方案：** NPU 不支持 colocated 模式，必须配置 `device_mapping` 使训练和推理在不同的 NPU 卡上执行。例如： - -```yaml -actor_train: - device_mapping: list(range(0, 4)) -actor_infer: - device_mapping: list(range(4, 8)) -``` - ### 不支持 Megatron 策略 **现象：** 在 NPU 上使用 `strategy: megatron` 配置时报错。 -**解决方案：** 当前提供的昇腾示例暂不支持 Megatron-LM 训练，请使用 DeepSpeed 作为训练后端： +**解决方案：** 昇腾 NPU 上不支持 Megatron-LM 训练，请使用 FSDP2 作为训练后端： ```yaml strategy_args: - strategy_name: deepspeed_train + strategy_name: fsdp2_train ``` ### HCCL 通信超时或失败 @@ -274,13 +261,6 @@ ulimit -n 65536 * hard nofile 65536 ``` -也可以在 ROLL YAML 配置中全局设置： - -```yaml -system_envs: - RAY_ULIMIT_NOFILE: "65536" -``` - ### NPU 显存不足 **现象：** 训练或推理过程中出现 OOM（Out of Memory）错误而崩溃。 @@ -289,11 +269,12 @@ system_envs: 1. 在配置文件中减小 `rollout_batch_size` 或 `num_return_sequences_in_group`。 2. 减小 `per_device_train_batch_size`，同时相应增大 `gradient_accumulation_steps`。 -3. 在配置中启用 DeepSpeed ZeRO-3 + CPU Offloading： +3. 在配置中启用 FSDP2 + CPU Offloading： ```yaml strategy_args: - strategy_name: deepspeed_train - strategy_config: ${deepspeed_zero3_cpuoffload} + strategy_name: fsdp2_train + strategy_config: + offload_policy: true ``` 4. 使用更小的模型或应用 LoRA 以降低显存占用。 @@ -303,7 +284,7 @@ system_envs: **解决方案：** -1. 确保 CANN 和 vLLM-Ascend 版本兼容（均应为 v0.13.0）。 +1. 确保 CANN 和 vLLM-Ascend 版本兼容（当前镜像使用 CANN 9.0.0 与 vLLM-Ascend v0.18）。 2. 检查 SOC 版本是否与硬件匹配。 3. 调整配置中 vLLM 的 `gpu_memory_utilization` 和 `max_model_len` 参数。 4. 确认已安装 `triton-ascend`（而非 `triton`），错误的 triton 后端会导致算子编译回退。 diff --git a/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_npu_rlvr.md b/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_npu_rlvr.md new file mode 100644 index 000000000..38b59331c --- /dev/null +++ b/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_npu_rlvr.md @@ -0,0 +1,822 @@ +# 在昇腾 NPU 上运行 RLVR 流水线 + +最后更新：2026/04/28。 + +本文档提供在华为昇腾 NPU 上运行 RLVR（Reinforcement Learning with Verifiable Rewards）流水线的端到端指南，涵盖环境准备、数据准备、模型下载、配置编写、训练启动、监控与评估，以及从 checkpoint 恢复训练。 + +## 工作流概览 + +从零开始在 NPU 上运行 RLVR 任务包含以下步骤： + +``` +1. 环境准备 → 2. 数据准备 → 3. 模型准备 → 4. 编写配置 → 5. 启动训练 → 6. 监控与评估 → 7. 从 Checkpoint 恢复 +``` + +## 步骤 1：环境准备 + +### 1.1 硬件与驱动前置条件 + +请确保硬件和宿主机驱动已经准备就绪： + +| 项目 | 要求 | +| ---- | ---- | +| 硬件 | Atlas 900 A2 PODc（Ascend 910B1）或 Atlas 900 A3 PODc（Ascend 910_9391） | +| 宿主机 OS | Ubuntu 22.04 | +| CANN | 9.0.0 | +| Ascend NPU 驱动 | 已在宿主机安装（`npu-smi info` 能看到设备） | +| Docker | >= 20.10 | + +### 1.2 获取 Docker 镜像 + +请使用与硬件匹配的预构建昇腾镜像。官方 ROLL NPU 镜像标签可在 https://quay.io/repository/ascend/roll?tab=tags 查看。容器启动细节参见 [Ascend NPU Docker 使用指南](ascend_docker_usage.md)。 + +```bash +# A2 硬件 +docker pull quay.io/ascend/roll:main-a2 +docker tag quay.io/ascend/roll:main-a2 roll:ascend-a2 + +# A3 硬件 +docker pull quay.io/ascend/roll:main-a3 +docker tag quay.io/ascend/roll:main-a3 roll:ascend-a3 +``` + +当前仓库提供 `docker/Dockerfile.A2` 和 `docker/Dockerfile.A3`，用于构建自定义镜像。如果你维护自定义镜像，请确保依赖版本与预构建镜像保持一致。 + +### 1.3 启动容器 + +```bash +docker run -dit \ + --name roll_npu \ + --ulimit nofile=65536:65536 \ + --device /dev/davinci0 \ + --device /dev/davinci1 \ + --device /dev/davinci2 \ + --device /dev/davinci3 \ + --device /dev/davinci4 \ + --device /dev/davinci5 \ + --device /dev/davinci6 \ + --device /dev/davinci7 \ + --device /dev/davinci_manager \ + --device /dev/devmm_svm \ + --device /dev/hisi_hdc \ + -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ + -v /usr/local/Ascend/add-ons:/usr/local/Ascend/add-ons \ + -v /usr/local/dcmi:/usr/local/dcmi \ + -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \ + -v /etc/ascend_install.info:/etc/ascend_install.info \ + -v /path/to/models:/data/models \ + -v /path/to/data:/data \ + --ipc=host \ + --net=host \ + roll:ascend-a3 \ + /bin/bash +``` + +> **注意：** `-v /path/to/models:/data/models` 和 `-v /path/to/data:/data` 分别挂载模型权重目录和训练数据目录。请根据实际环境调整路径。 + +### 1.4 验证环境 + +进入容器后，执行： + +```bash +# 验证 NPU 可见性 +npu-smi info + +# 验证 CANN 环境是否已加载 +env | grep -E "ASCEND|LD_LIBRARY_PATH|PATH" + +# 验证 Python 包 +python -c "import torch; import torch_npu; print(torch.npu.is_available())" +python -c "import vllm; print(f'vllm: {vllm.__version__}')" +python -c "import vllm_ascend; print(f'vllm_ascend available')" +``` + +如果以上验证均通过，说明环境已经准备就绪。环境变量的详细说明参见 [NPU 环境配置指南](ascend_npu_env_config.md)。 + +## 步骤 2：数据准备 + +RLVR 流水线使用 JSONL 格式的数据文件。不同奖励领域需要不同的数据字段。 + +### 2.1 数据格式 + +#### 通用字段（所有领域都必需） + +| 字段 | 类型 | 是否必需 | 说明 | +| ---- | ---- | -------- | ---- | +| `id` | string/int | 是 | 数据样本的唯一标识 | +| `messages` 或 `prompt` | string | 是 | 输入 prompt；`messages` 是消息列表的 JSON 字符串 | +| `tag` | string | 是 | 奖励领域标签，用于决定使用哪个 Reward Worker | + +#### 领域专属字段 + +| 领域 | tag 值 | 必需字段 | 说明 | +| ---- | ------ | -------- | ---- | +| 数学规则 | `math_rule` | `ground_truth` | 正确答案 | +| 代码沙箱 | `code_sandbox`（如 `KodCode`） | `test_cases`, `case_type` | 测试用例和类型（如 `pytest`） | +| LLM Judge | `llm_judge`（如 `RLVR`） | `ground_truth` | 参考答案或参考回复 | +| IFEval | `ifeval` | 无额外字段 | 基于规则的指令遵循评估 | +| CrossThinkQA | `crossthinkqa` | `ground_truth` | 跨学科推理答案 | + +#### 数据样例 + +**数学领域（math_rule）：** + +```json +{ + "id": "0", + "source": "gsm8k", + "difficulty": 0, + "prompt": "Solve the equation 3x + 5 = 14", + "messages": "[{\"role\": \"system\", \"content\": \"You are a math assistant.\"}, {\"role\": \"user\", \"content\": \"Solve the equation 3x + 5 = 14\"}]", + "ground_truth": "3", + "tag": "math_rule" +} +``` + +**代码领域（code_sandbox）：** + +```json +{ + "id": "5ea1ab", + "source": "codeforces", + "difficulty": "0", + "prompt": "Write a function that takes an array of distinct integers and returns all possible permutations.", + "messages": "[{\"role\": \"user\", \"content\": \"Write a function...\"}]", + "ground_truth": "[\"def permute(nums): ...\"]", + "case_type": "pytest", + "test_case_function": "", + "test_cases": "[{\"assert_code\": \"def test_permute(): ...\"}]", + "tag": "KodCode" +} +``` + +### 2.2 数据放置 + +将数据文件放到容器内的某个目录中（如 `/data/`），并在 `actor_train.data_args` 中指定路径： + +```yaml +actor_train: + data_args: + file_name: + - data/math_deepmath_deal.jsonl + - data/code_KodCode_data.jsonl + dataset_dir: data +``` + +### 2.3 验证数据 + +验证数据用于训练过程中的周期性评估。请在 `validation` 配置中指定： + +```yaml +validation: + data_args: + template: qwen2_5 + file_name: + - data/math_benchmarks.jsonl + generating_args: + max_new_tokens: ${response_length} + top_p: 0.6 + temperature: 0.6 + num_return_sequences: 1 +``` + +验证数据中的 `tag` 字段应与训练数据中的 tag 保持一致，这样才能按领域报告准确率。 + +## 步骤 3：模型准备 + +### 3.1 下载模型权重 + +RLVR 流水线需要以下模型： + +| 模型 | 配置键 | 说明 | +| ---- | ------ | ---- | +| Actor / Reference 模型 | `pretrain` | 用于训练和推理的策略模型 | +| Reward 模型 | `reward_pretrain` | Reward Worker 中使用的模型（如数学规则奖励中的答案抽取） | + +以 Qwen2.5-7B 为例： + +```bash +# 使用 ModelScope 下载（推荐中国大陆用户使用） +pip install modelscope +modelscope download --model Qwen/Qwen2.5-7B --local_dir /data/models/Qwen2.5-7B + +# 或使用 HuggingFace 下载 +huggingface-cli download Qwen/Qwen2.5-7B --local-dir /data/models/Qwen2.5-7B +``` + +### 3.2 在配置中指定模型路径 + +```yaml +pretrain: Qwen/Qwen2.5-7B # 从 ModelScope/HuggingFace 自动下载 +# 或使用本地路径 +# pretrain: /data/models/Qwen2.5-7B + +reward_pretrain: Qwen/Qwen2.5-7B +``` + +> **提示：** 如果容器内网络访问受限，请提前在宿主机下载模型，通过 `-v` 挂载到容器中，并在配置中使用本地路径。 + +## 步骤 4：编写 NPU 配置 + +### 与 GPU 的关键差异 + +将 GPU RLVR 配置适配到 NPU 时，**必须**进行以下修改： + +| 项目 | GPU | NPU | +| ---- | --- | --- | +| 训练后端 | Megatron 或 FSDP2 | 仅 FSDP2（NPU 不支持 Megatron） | +| 推理后端 | vLLM | vLLM-Ascend | +| Reference 模型策略 | `megatron_infer` | `fsdp2_infer` | +| 注意力实现 | `flash_attn` 或 `fa2` | 通过 `transformers` 使用 `fa2`（不能使用 `flash_attn` 包） | +| 通信后端 | NCCL | HCCL | +| 设备可见性 | `CUDA_VISIBLE_DEVICES` | `ASCEND_RT_VISIBLE_DEVICES` | +| 分片配置 | FSDP2 或 Megatron 优化器分片 | FSDP2，7B+ 模型推荐 `offload_policy: true` | + +### 完整 NPU 配置样例 + +下面是一个完整的 NPU 适配配置（改编自 `examples/ascend_examples/qwen3_30b_rlvr_fsdp2.yaml`），关键差异使用 `# NPU` 注释标记： + +```yaml +hydra: + run: + dir: . + output_subdir: null + +exp_name: "qwen3-30BA3B-rlvr-npu" +seed: 42 +logging_dir: ./output/logs +output_dir: ./output +system_envs: + USE_MODELSCOPE: '1' + HCCL_NPU_SOCKET_PORT_RANGE: auto # NPU：允许同卡多进程 HCCL 自动分配 device 侧端口 + VLLM_ASCEND_ENABLE_NZ: '0' # NPU：RL 权重刷新场景需禁用 FRACTAL_NZ + +checkpoint_config: + type: file_system + output_dir: ./output/models/${exp_name} + +track_with: tensorboard +tracker_kwargs: + log_dir: ./output/tensorboard/rlvr_npu +rpc_timeout: 72000 + +num_gpus_per_node: 16 + +max_steps: 500 +save_steps: 100 +logging_steps: 1 +eval_steps: 10 +resume_from_checkpoint: false + +rollout_batch_size: 32 +prompt_length: 2048 +response_length: 4096 +num_return_sequences_in_group: 8 + +ppo_epochs: 1 +adv_estimator: "reinforce" + +value_clip: 0.5 +reward_clip: 10 +advantage_clip: 2.0 +dual_clip_loss: true + +norm_mean_type: ~ +norm_std_type: ~ + +max_len_mask: true +difficulty_mask: true +difficulty_low_threshold: 0.1 +difficulty_high_threshold: 0.95 +error_max_len_clip: false + +difficulty_loss_weight: false +length_loss_weight: false + +add_token_level_kl: false +whiten_advantages: true + +pretrain: Qwen/Qwen3-30B-A3B +reward_pretrain: Qwen/Qwen3-30B-A3B + +actor_train: + model_args: + disable_gradient_checkpointing: false + dtype: bf16 + model_type: ~ + training_args: + learning_rate: 1.0e-6 + weight_decay: 0 + per_device_train_batch_size: 2 + gradient_accumulation_steps: 8 + warmup_steps: 20 + num_train_epochs: 50 + data_args: + template: qwen2_5 + file_name: + - data/math_deepmath_deal.jsonl + domain_interleave_probs: + math_rule: 1 + dataset_dir: data + messages: messages + interleave_probs: "1.0" + preprocessing_num_workers: 16 + strategy_args: + strategy_name: fsdp2_train # NPU：必须使用 FSDP2，不能用 megatron_train + strategy_config: + fsdp_size: 16 # NPU：FSDP2 分片大小 + param_dtype: bf16 + reduce_dtype: bf16 + offload_policy: true # NPU：大模型启用 CPU offloading + apply_expert_patch: true # NPU：MoE 模型必须启用 + apply_tiled_mlp: true # NPU：TiledMLP 降低显存 + tiled_num_shards: 8 + reshard_after_forward: true + wrap_policy: # NPU：MoE 专用 wrap policy + wrap_embeddings: true + wrap_lm_output: true + moe_experts: + - Qwen3MoeMLP + transformer_layer_cls_to_wrap: + - Qwen3MoeAttention + - Qwen3MoeSparseMoeBlock + use_remove_padding: true + device_mapping: list(range(0,16)) # NPU：训练使用 NPU 0-15 + infer_batch_size: 2 + +actor_infer: + model_args: + disable_gradient_checkpointing: true + dtype: bf16 + generating_args: + max_new_tokens: ${response_length} + top_p: 0.99 + top_k: 100 + num_beams: 1 + temperature: 0.99 + num_return_sequences: ${num_return_sequences_in_group} + data_args: + template: qwen2_5 + strategy_args: + strategy_name: vllm # NPU：使用 vLLM-Ascend 推理 + strategy_config: + gpu_memory_utilization: 0.8 + block_size: 16 + max_model_len: 6144 + tensor_parallel_size: 2 + enforce_eager: true + load_format: dummy + device_mapping: list(range(0,16)) # NPU：推理与训练共享 NPU + infer_batch_size: 1 + +reference: + model_args: + disable_gradient_checkpointing: true + dtype: bf16 + model_type: ~ + data_args: + template: qwen2_5 + strategy_args: + strategy_name: fsdp2_infer # NPU：使用 fsdp2_infer，不能用 megatron_infer + strategy_config: + fsdp_size: 16 + param_dtype: bf16 + reduce_dtype: bf16 + apply_tiled_mlp: true + tiled_num_shards: 8 + reshard_after_forward: true + offload_policy: true + device_mapping: list(range(0,16)) # NPU：Reference 与训练共享 NPU + infer_batch_size: 2 + +rewards: + math_rule: + worker_cls: roll.pipeline.rlvr.rewards.math_rule_reward_worker.MathRuleRewardWorker + model_args: + model_name_or_path: ${reward_pretrain} + data_args: + template: qwen2_5 + tag_included: [deepmath_103k, aime] + world_size: 8 + infer_batch_size: 1 +``` + +### 关键配置变更说明 + +#### 1. 训练策略：使用 FSDP2 替代 Megatron + +```yaml +# GPU（原始配置） +actor_train: + strategy_args: + strategy_name: megatron_train + strategy_config: + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 1 + +# NPU（适配后） +actor_train: + strategy_args: + strategy_name: fsdp2_train + strategy_config: + fsdp_size: 4 + param_dtype: bf16 + reduce_dtype: bf16 + reshard_after_forward: true + offload_policy: true +``` + +在 4 张 NPU 上运行 7B 模型时，设置 `offload_policy: true` 可以启用 CPU offloading 避免 OOM。对于更小的模型（如 0.5B），`offload_policy: false` 可能已经足够。 + +#### 2. Reference 模型：使用 fsdp2_infer 替代 megatron_infer + +```yaml +# GPU +reference: + strategy_args: + strategy_name: megatron_infer + +# NPU +reference: + strategy_args: + strategy_name: fsdp2_infer + strategy_config: + fsdp_size: 4 + param_dtype: bf16 + reduce_dtype: bf16 + reshard_after_forward: true + offload_policy: true +``` + +#### 3. 注意力实现 + +通过 `transformers` 库使用 `fa2`，不要使用 `flash_attn` 包： + +```yaml +actor_train: + model_args: + attn_implementation: fa2 # 不能使用 flash_attn +``` + +#### 4. 系统环境变量 + +ROLL 会为 worker 注入设备可见性和 Ray 运行时变量，但生产运行时仍建议显式设置 HCCL、显存、vLLM-Ascend、缓存和日志相关变量。推荐的单机和多机环境变量设置参见 [NPU 环境配置指南](ascend_npu_env_config.md)。 + +## 步骤 5：启动训练 + +### 单机 + +运行仓库中提供的昇腾 RLVR 示例： + +```bash +cd /workspace/ROLL +export PYTHONPATH="/workspace/ROLL:$PYTHONPATH" + +python examples/start_rlvr_pipeline.py \ + --config_path ascend_examples \ + --config_name qwen3_30b_rlvr_fsdp2 +``` + +如果你将上面的自定义配置保存为 `/rlvr_npu.yaml`，则使用 `--config_path --config_name rlvr_npu`。 + +### 多机 + +对于跨多个昇腾 NPU 节点的多机训练，ROLL 通过环境变量提供自动 Ray 集群管理。 + +#### 设置 + +启动前，请在**每个**节点上设置以下环境变量。请将占位符替换为实际值： + +**Head 节点（RANK=0）：** + +```bash +# Ray 集群 +export RANK=0 +export WORLD_SIZE=2 +export MASTER_ADDR=10.0.0.1 # Head 节点 IP +export MASTER_PORT=6379 +export DASHBOARD_PORT=8265 + +# HCCL 多机通信 +export HCCL_CONNECT_TIMEOUT=3600 +export HCCL_EXEC_TIMEOUT=3600 +export HCCL_DETERMINISTIC=false +export HCCL_OP_EXPANSION_MODE="AIV" +export HCCL_NPU_SOCKET_PORT_RANGE="auto" +export HCCL_IF_IP=10.0.0.1 # 当前节点 IP +export HCCL_SOCKET_IFNAME="enp194s0f0" # HCCL 网络接口 +export HCCL_IF_BASE_PORT=23456 + +# vLLM-Ascend RL 场景 +export VLLM_ASCEND_ENABLE_NZ=0 + +# NPU 显存、CPU、vLLM、缓存、日志等（同单机设置） +# 完整列表参见 NPU 环境配置指南 +``` + +**Worker 节点（RANK=1）：** + +```bash +# Ray 集群 +export RANK=1 +export WORLD_SIZE=2 +export MASTER_ADDR=10.0.0.1 # Head 节点 IP（同上） +export MASTER_PORT=6379 +export DASHBOARD_PORT=8265 + +# HCCL 多机通信 +export HCCL_CONNECT_TIMEOUT=3600 +export HCCL_EXEC_TIMEOUT=3600 +export HCCL_DETERMINISTIC=false +export HCCL_OP_EXPANSION_MODE="AIV" +export HCCL_NPU_SOCKET_PORT_RANGE="auto" +export HCCL_IF_IP=10.0.0.2 # 当前节点 IP +export HCCL_SOCKET_IFNAME="enp194s0f0" +export HCCL_IF_BASE_PORT=23456 + +# vLLM-Ascend RL 场景 +export VLLM_ASCEND_ENABLE_NZ=0 + +# NPU 显存、CPU、vLLM、缓存、日志等（同单机设置） +``` + +#### 启动 + +在所有节点运行**相同**命令。ROLL 会读取 `RANK`，决定当前进程以 head 还是 worker 方式启动。 + +运行这些命令前，请先将多机配置保存为 `/rlvr_npu_multinode.yaml`。 + +**在 Head 节点上：** + +```bash +cd /workspace/ROLL +export PYTHONPATH="/workspace/ROLL:$PYTHONPATH" + +python examples/start_rlvr_pipeline.py \ + --config_path \ + --config_name rlvr_npu_multinode +``` + +**在每个 Worker 节点上：** + +```bash +cd /workspace/ROLL +export PYTHONPATH="/workspace/ROLL:$PYTHONPATH" + +python examples/start_rlvr_pipeline.py \ + --config_path \ + --config_name rlvr_npu_multinode +``` + +Worker 节点会输出已加入集群的日志，然后退出（`sys.exit(0)`）。对应的 Ray 进程会保持存活，用于服务训练任务。Head 节点会继续执行完整训练流水线。 + +:::tip +也可以在运行流水线前手动预启动 Ray（head 节点执行 `ray start --head`，worker 节点执行 `ray start --address=...`）。ROLL 会检测已有集群并跳过自动启动。 +::: + +#### 验证集群 + +在 Head 节点上检查所有节点是否已经加入： + +```bash +ray status +``` + +输出中应能看到来自所有节点的 NPU 资源。例如，2 节点 × 8 NPU： + +``` +Resources +--------------------------------------------------------------- +Total: 128.0 CPU, 16.0 NPU, ... +``` + +#### 多机配置 + +对于多机配置，需要调整 `device_mapping` 以覆盖跨节点的 NPU。例如，2 节点 × 8 NPU： + +```yaml +num_gpus_per_node: 8 + +# 训练在 Node0 的 NPU 0-7 上执行 +actor_train: + strategy_args: + strategy_name: fsdp2_train + strategy_config: + fsdp_size: 8 + param_dtype: bf16 + reduce_dtype: bf16 + reshard_after_forward: true + offload_policy: true + device_mapping: list(range(0,8)) + +# 推理在 Node1 的 NPU 0-7 上执行 +actor_infer: + strategy_args: + strategy_name: vllm + strategy_config: + gpu_memory_utilization: 0.8 + max_model_len: 8000 + device_mapping: list(range(8,16)) + +# Reference 模型共享推理 NPU +reference: + strategy_args: + strategy_name: fsdp2_infer + strategy_config: + fsdp_size: 8 + param_dtype: bf16 + reduce_dtype: bf16 + reshard_after_forward: true + offload_policy: true + device_mapping: list(range(8,16)) +``` + +包含数据准备和 Reward Worker 的完整多机配置样例参见 [NPU 端到端配置样例](ascend_npu_examples.md#样例-3多机分布式训练)。 + +#### 多机重要注意事项 + +- **需要共享存储：** 模型权重、训练数据和 checkpoint 必须能在所有节点以相同路径访问。请将 NFS 或其他共享文件系统挂载到每个容器中。 +- **网络要求：** 所有节点必须位于同一二层网络中。所有 worker 节点都必须能访问 head 节点的 6379 端口。 +- **HCCL 网络接口：** 所有节点上的 `HCCL_SOCKET_IFNAME` 必须一致，并且应对应高速互联网络（如 RoCE）。可使用 `ip addr` 或 `hccn_tool` 识别正确网卡。 + +## 步骤 6：监控与评估 + +### 6.1 训练监控 + +ROLL 内置支持 TensorBoard。请在配置中指定日志目录： + +```yaml +track_with: tensorboard +tracker_kwargs: + log_dir: ./output/tensorboard/rlvr_npu +``` + +启动 TensorBoard： + +```bash +tensorboard --logdir ./output/tensorboard/rlvr_npu --port 6006 +``` + +建议重点监控以下指标： + +| 指标 | 说明 | +| ---- | ---- | +| `time/step_total` | 每步总耗时 | +| `time/step_generate` | 推理生成耗时 | +| `time/step_train` | 训练更新耗时 | +| `train/loss` | 训练损失 | +| `train/lr` | 当前学习率 | +| `reward/mean` | 平均奖励 | +| `response_length/mean` | 平均生成长度 | + +### 6.2 验证评估 + +流水线会按 `eval_steps` 间隔自动运行验证评估。验证结果包括： + +| 指标 | 说明 | +| ---- | ---- | +| `val_correct/all/mean` | 所有验证样本的准确率 | +| `val_correct//mean` | 每个 tag 分组的准确率（如 `val_correct/math_rule/mean`） | + +验证准确率是衡量 RLVR 训练效果的核心指标。随着训练推进，该指标通常应逐步提升。 + +### 6.3 生成样例 + +训练过程中，每隔 `logging_steps` 步会将生成样例打印到日志中，便于直观评估模型输出质量。 + +## 步骤 7：从 Checkpoint 恢复 + +### 7.1 Checkpoint 保存 + +流水线会按照 `save_steps` 间隔，自动将 checkpoint 保存到 `checkpoint_config.output_dir`： + +```yaml +checkpoint_config: + type: file_system + output_dir: /data/models/${exp_name} + +save_steps: 100 +``` + +### 7.2 从 Checkpoint 恢复 + +将 `resume_from_checkpoint` 设置为 checkpoint 路径即可恢复训练： + +```yaml +resume_from_checkpoint: ./output/models/qwen3-30BA3B-rlvr-npu/checkpoint-100 +``` + +或者在启动命令中覆盖该参数： + +```bash +python examples/start_rlvr_pipeline.py \ + --config_path \ + --config_name rlvr_npu \ + resume_from_checkpoint=./output/models/qwen3-30BA3B-rlvr-npu/checkpoint-100 +``` + +## 设备映射参考 + +以下是 RLVR 常见的 NPU 资源分配模式。可以根据工作负载和硬件情况选择共卡模式（训练和推理共享 NPU）或分离模式： + +### 8 卡单机（7B 模型） + +| 组件 | NPU | 数量 | 说明 | +| ---- | --- | ---- | ---- | +| actor_train | 0-3 | 4 | FSDP2 + CPU offloading | +| actor_infer | 4-7 | 4 | vLLM-Ascend | +| reference | 4-7（共享） | - | fsdp2_infer，与 actor_infer 共享 | +| reward workers | CPU | - | 数学规则和代码沙箱运行在 CPU 上 | + +### 16 卡单机（A3，7B 模型） + +| 组件 | NPU | 数量 | 说明 | +| ---- | --- | ---- | ---- | +| actor_train | 0-7 | 8 | FSDP2 | +| actor_infer | 8-15 | 8 | vLLM-Ascend | +| reference | 8-15（共享） | - | fsdp2_infer，与 actor_infer 共享 | +| reward workers | CPU | - | 数学规则和代码沙箱运行在 CPU 上 | + +### 2×8 卡多机（7B 模型） + +| 组件 | NPU | 数量 | 说明 | +| ---- | --- | ---- | ---- | +| actor_train | Node0: 0-7 | 8 | FSDP2 + CPU offloading | +| actor_infer | Node1: 0-7 | 8 | vLLM-Ascend | +| reference | Node1: 0-7（共享） | - | fsdp2_infer，与 actor_infer 共享 | +| reward workers | CPU | - | 数学规则和代码沙箱运行在 CPU 上 | + +## NPU 上支持的 Reward Worker + +NPU 上支持以下 RLVR Reward Worker： + +| Reward Worker | 类 | NPU 兼容性 | 说明 | +| ------------- | --- | ---------- | ---- | +| Math Rule Reward | `MathRuleRewardWorker` | ✅ 支持 | 基于规则评估，运行在 CPU 上 | +| Code Sandbox Reward | `CodeSandboxRewardWorker` | ✅ 支持 | 在沙箱中执行代码，运行在 CPU 上 | +| LLM Judge Reward | `LLMJudgeRewardWorker` | ✅ 支持 | 需要额外 NPU 用于 judge 模型推理 | +| IFEval Rule Reward | `GeneralRuleRewardWorker` | ✅ 支持 | 基于规则评估，运行在 CPU 上 | +| CrossThinkQA Reward | `CrossThinkQARuleRewardWorker` | ✅ 支持 | 基于规则评估，运行在 CPU 上 | + +:::caution +使用 `LLMJudgeRewardWorker` 时，judge 模型需要独立的 NPU 设备进行推理。请确保在 `device_mapping` 中为 judge 模型分配独立 NPU，不要与 `actor_infer` 或 `actor_train` 共享。 +::: + +## GPU 到 NPU 配置迁移 Checklist + +将已有 GPU RLVR 配置迁移到 NPU 时，可使用以下 checklist： + +- [ ] 将 `actor_train.strategy_args.strategy_name` 从 `megatron_train` 改为 `fsdp2_train` +- [ ] 将 `actor_train.strategy_args.strategy_config` 改为 FSDP2 配置（7B+ 模型使用 `offload_policy: true`） +- [ ] 将 `reference.strategy_args.strategy_name` 从 `megatron_infer` 改为 `fsdp2_infer` +- [ ] 将 `reference.strategy_args.strategy_config` 设置为与 `actor_train` 一致的 FSDP2 配置 +- [ ] 在 `actor_train.model_args` 和 `reference.model_args` 中添加 `attn_implementation: fa2` +- [ ] 移除所有 `flash_attn` 引用 +- [ ] 移除所有 Megatron 专属配置（如 `tensor_model_parallel_size`、`pipeline_model_parallel_size`） +- [ ] 如果使用 `llm_judge` reward worker，确认它有独立的 NPU 分配 + +## 常见问题 + +### 首次推理请求极慢 + +模型加载后的首次推理请求会触发算子编译，可能需要几分钟。这是一次性开销。可通过以下方式缓解： + +1. 启用算子编译缓存（参见 [NPU 环境配置指南](ascend_npu_env_config.md) 中的 `ACL_OP_COMPILER_CACHE_MODE`）。 +2. 在正式训练循环开始前运行一次 warmup 请求。 + +### 7B 模型在 4 张 NPU 上 OOM + +如果在 4 张 NPU 上运行 7B 模型时遇到 OOM： + +1. 切换到 `fsdp2_train` 策略并设置 `offload_policy: true`。 +2. 将 `per_device_train_batch_size` 降到 1。 +3. 相应增大 `gradient_accumulation_steps`。 +4. 降低 vLLM 配置中的 `max_model_len`（如从 8192 降到 4096）。 + +### HCCL 通信超时 + +参见 FAQ 中的 [HCCL 通信超时或失败](ascend_npu_faq.md#hccl-通信超时或失败)。 + +### vLLM-Ascend 导入错误 + +请确认 CANN 环境已经正确 source： + +```bash +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +``` + +### triton 冲突 + +在 NPU 上，`triton` 包会与 `triton-ascend` 冲突。可通过以下方式修复： + +```bash +pip uninstall -y triton triton-ascend +pip install triton-ascend==3.2.1 --extra-index-url https://mirrors.huaweicloud.com/ascend/repos/pypi +``` + +更多排障建议参见 [Ascend NPU FAQ](ascend_npu_faq.md)。 + +## 声明 + +ROLL 中提供的 Ascend 支持代码皆为参考样例，生产环境使用请通过官方正式途径沟通。 diff --git a/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_usage.md b/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_usage.md index 54d45fb81..16f32ba9a 100644 --- a/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_usage.md +++ b/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_usage.md @@ -1,26 +1,44 @@ # ROLL x Ascend -Last updated: 11/25/2025. +最后更新：2026/06/23。 我们在 ROLL 上增加对华为昇腾设备的支持。 -## 硬件支持 +## 硬件配套和支持的操作系统 -Atlas 900 A2 PODc +ROLL 昇腾适配当前覆盖以下训练系列硬件： +| 产品 | 是否支持 | 说明 | +| ---- | -------- | ---- | +| Atlas 900 A2 PODc（Ascend 910B1）/ Atlas A2 训练系列产品 | √ | 使用 `docker/Dockerfile.A2` 或 `roll:ascend-a2` 镜像。 | +| Atlas 900 A3 PODc（Ascend 910_9391）/ Atlas A3 训练系列产品 | √ | 使用 `docker/Dockerfile.A3` 或 `roll:ascend-a3` 镜像。 | +| Atlas A5 训练系列产品 | √ | 使用 A5 安装配置：torch 2.10、vLLM v0.20.2、vLLM-Ascend `main`，并在构建 vLLM-Ascend 时设置 `COMPILE_CUSTOM_KERNELS=1`。 | +| Atlas A2/A3 推理系列产品、Atlas 200I/500 A2 推理产品 | x | 当前 ROLL NPU 镜像和示例面向训练系列设备。 | +| 其他昇腾训练或推理产品 | 未验证 | 使用前请确认驱动、固件、CANN、`torch_npu` 与 vLLM-Ascend 版本配套。 | -## 安装 +> 本节表格中 `√` 代表当前 ROLL 昇腾 Dockerfile、示例或 A5 手动安装配置已支持，`x` 代表当前 ROLL NPU 配套不支持。 + +支持的操作系统： +| 部署场景 | 支持的操作系统 | 说明 | +| -------- | -------------- | ---- | +| 物理机宿主机 | Ubuntu 22.04 | 当前 ROLL 昇腾文档推荐并验证的宿主机操作系统。 | +| ROLL 昇腾容器 | Ubuntu 22.04 | A2/A3 Dockerfile 基于 `quay.io/ascend/cann:9.0.0-*-ubuntu22.04-py3.11`。 | +| Atlas A5 手动安装 | Ubuntu 22.04 | 使用下文 A5 专用 torch/vLLM 版本组合。驱动、固件、CANN 和 `torch_npu` 版本需要与目标 A5 环境匹配。 | +| 其他宿主机 OS 上的虚拟机或容器部署 | 以昇腾/CANN 兼容性说明为准 | 请结合目标硬件查询昇腾兼容性查询助手，以及 CANN 软件安装文档中的操作系统兼容性说明。 | + +## 安装 ### 基础环境准备 -| software | version | +| 软件 | 版本 | |-----------|-------------| | Python | 3.11 | -| CANN | 8.3.RC1 | +| CANN | 9.0.0 | -### 创建 conda 环境 +Atlas A5 请保持 Python 3.11，并使用下文 [A5 安装配置](#a5-安装配置) 中的专用 torch/vLLM 版本组合。 +### 创建 conda 环境 使用以下命令在 Miniconda 中创建新的 conda 环境： @@ -29,27 +47,25 @@ conda create --name roll python=3.11 conda activate roll ``` -### 安装 torch & torch_npu: +### 安装 torch & torch_npu - -为了能在 ROLL 中正常使用 torch 和 torch_npu，需使用以下命令安装 torch 和 torch_npu。 +为了能在 ROLL 中正常使用 torch 和 torch_npu，需使用以下命令安装 torch 和 torch_npu： ``` -# 安装 torch 的 CPU 版本 -pip install torch==2.7.1 torchvision==0.22.1 torchaudio==2.7.1 --index-url https://download.pytorch.org/whl/cpu +# 在预构建镜像外手动安装时，使用 CPU 版 torch +pip install torch==2.9.0 torchvision==0.24.0 torchaudio==2.9.0 --index-url https://download.pytorch.org/whl/cpu -# 安装 torch_npu -pip install torch_npu==2.7.1 +# 安装与 torch/CANN 匹配的 torch_npu +pip install torch_npu==2.9.0 ``` +### 安装 vllm & vllm-ascend -### 安装vllm & vllm-ascend: - -为了能够在 ROLL 中正常使用 vllm，需使用以下命令编译安装 vllm 和 vllm-ascend。 +为了能够在 ROLL 中正常使用 vllm，需使用以下命令编译安装 vllm 和 vllm-ascend： ``` # vllm -git clone -b v0.11.0 --depth 1 https://github.com/vllm-project/vllm.git +git clone -b v0.18.0 --depth 1 https://github.com/vllm-project/vllm.git cd vllm pip install -r requirements/build.txt @@ -57,7 +73,7 @@ VLLM_TARGET_DEVICE=empty pip install -v -e . cd .. # vllm-ascend -git clone -b v0.11.0rc1 --depth 1 https://github.com/vllm-project/vllm-ascend.git +git clone -b v0.18.0 --depth 1 https://github.com/vllm-project/vllm-ascend.git cd vllm-ascend pip install -e . @@ -65,12 +81,39 @@ cd .. ``` 或者可以从预编译的 wheel 包安装 `vllm` 和 `vllm-ascend`： + ``` -# Install vllm-project/vllm. The newest supported version is v0.11.0. -pip install vllm==0.11.0 +# 安装 vllm-project/vllm，最新支持版本为 v0.18.0 +pip install vllm==0.18.0 -# Install vllm-project/vllm-ascend from pypi. -pip install vllm-ascend==0.11.0rc1 +# 从 pypi 安装 vllm-project/vllm-ascend +pip install vllm-ascend==0.18 +``` + +### A5 安装配置 + +Atlas A5 上使用 torch 2.10、vLLM v0.20.2，并从 `main` 分支安装 vLLM-Ascend。安装 vLLM-Ascend 前需要设置 `COMPILE_CUSTOM_KERNELS=1`，以便编译自定义 kernel： + +``` +# 安装 torch 2.10 +pip install torch==2.10.0 torchvision==0.25.0 torchaudio==2.10.0 --index-url https://download.pytorch.org/whl/cpu + +# 安装与 torch 2.10 和 CANN 版本匹配的 torch_npu +pip install torch_npu==2.10.0 + +# vLLM v0.20.2 +git clone -b v0.20.2 --depth 1 https://github.com/vllm-project/vllm.git +cd vllm +pip install -r requirements/build.txt +VLLM_TARGET_DEVICE=empty pip install -v -e . +cd .. + +# vLLM-Ascend main +git clone -b main --depth 1 https://github.com/vllm-project/vllm-ascend.git +cd vllm-ascend +export COMPILE_CUSTOM_KERNELS=1 +pip install -v -e . +cd .. ``` ### 安装 ROLL @@ -79,59 +122,59 @@ pip install vllm-ascend==0.11.0rc1 git clone https://github.com/alibaba/ROLL.git cd ROLL pip install -r requirements_common.txt +pip install -e . cd .. ``` ### 其他三方库说明 -| software | description | -|-------------------------------|---------------| -| transformers | >= v4.57.1 | -| flash_attn | not supported | -| transformer-engine[pytorch] | not supported | +| 软件 | 说明 | +| ---- | ---- | +| transformers | >= v4.57.6 | +| flash_attn | 不支持 | +| transformer-engine[pytorch] | 不支持 | -1. 支持通过 transformers 使能 --flash_attention_2， transformers 需大于等于 4.57.1 版本。 -2. 不支持通过 flash_attn 使能 flash attention 加速。 -3. 暂不支持 transformer-engine[pytorch] +1. `transformers` v4.57.6 支持启用 `--flash_attention_2`。 +2. 目前不支持 `flash_attn` 加速。 +3. 目前不支持 `transformer-engine[pytorch]`。 ``` -pip install transformers==4.57.1 +pip install transformers==4.57.6 ``` -## 快速开始，单节点部署指引 +## 快速开始：单节点部署指引 正式使用前，建议您通过对单节点流水线的训练尝试以检验环境准备和安装的正确性。 -由于目前暂不支持 Megatron-LM 训练，请首先将对应文件中 -strategy_args 参数修改为 fsdp2 选项。 +由于 NPU 上不支持 Megatron-LM 训练，请首先将对应文件中 `strategy_args` 参数修改为 `fsdp2` 选项。 -1. 使用 shell 执行单节点流水线 + +1. 使用 shell 执行单节点流水线： ``` bash examples/agentic_demo/run_agentic_pipeline_frozen_lake_single_node_demo.sh ``` -2. 使用配置文件执行 agentic pipeline +2. 使用配置文件执行 agentic pipeline： ``` -# 确保当前位于ROLL项目目录的根目录下 +# 确保当前位于 ROLL 项目目录的根目录下 python examples/start_agentic_pipeline.py \ --config_path qwen2.5-0.5B-agentic \ --config_name agentic_val_sokoban - -- ``--config_path`` – 包含您的YAML配置文件的目录。 -- ``--config_name`` – 文件名（不含.yaml后缀）。 ``` -## 支持现状 +- `--config_path` – 包含您的 YAML 配置文件的目录。 +- `--config_name` – 文件名（不含 `.yaml` 后缀）。 -| Feature | Example | Training Backend | Inference Backend | Hardware | -| --------------- | ------------------------------------------------------------ | ---------------- | ----------------- | ----------------- | -| Agentic | examples/qwen2.5-0.5B-agentic/run_agentic_pipeline_sokoban.sh | FSDP2 | vLLM | Atlas 900 A2 PODc | -| Agentic-Rollout | examples/qwen2.5-0.5B-agentic/run_agentic_rollout_sokoban.sh | FSDP2 | vLLM | Atlas 900 A2 PODc | -| DPO | examples/qwen2.5-3B-dpo_megatron/run_dpo_pipeline.sh | FSDP2 | vLLM | Atlas 900 A2 PODc | -| RLVR | examples/qwen2.5-7B-rlvr_megatron/run_rlvr_pipeline.sh | FSDP2 | vLLM | Atlas 900 A2 PODc | +## 支持现状 +| 功能 | 示例 | 训练后端 | 推理后端 | 硬件 | +| ---- | ---- | -------- | -------- | ---- | +| Agentic | examples/qwen2.5-0.5B-agentic/run_agentic_pipeline_sokoban.sh | FSDP2 | vLLM | Atlas 900 A2/A3 PODc | +| Agentic-Rollout | examples/qwen2.5-0.5B-agentic/run_agentic_rollout_sokoban.sh | FSDP2 | vLLM | Atlas 900 A2/A3 PODc | +| RLVR | examples/ascend_examples/run_rlvr_pipeline.sh | FSDP2 | vLLM | Atlas 900 A2/A3/A5 训练系列 | ## 声明 -ROLL 中提供的 Ascend 支持代码皆为参考样例，生产环境使用请通过官方正式途径沟通，谢谢。 + +ROLL 中提供的 Ascend 支持代码皆为参考样例，生产环境使用请通过官方正式途径沟通。 diff --git a/examples/ascend_examples/run_dpo_pipeline.sh b/examples/ascend_examples/run_dpo_pipeline.sh deleted file mode 100644 index cc5a049f4..000000000 --- a/examples/ascend_examples/run_dpo_pipeline.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -set +x - -CONFIG_PATH=$(basename $(dirname $0)) -python examples/start_dpo_pipeline.py --config_path $CONFIG_PATH --config_name qwen3_4B_dpo_deepspeed diff --git a/examples/ascend_examples/run_rlvr_pipeline.sh b/examples/ascend_examples/run_rlvr_pipeline.sh index 770bd9340..e7dec4725 100644 --- a/examples/ascend_examples/run_rlvr_pipeline.sh +++ b/examples/ascend_examples/run_rlvr_pipeline.sh @@ -5,4 +5,4 @@ export HCCL_NPU_SOCKET_PORT_RANGE="auto" export VLLM_ASCEND_ENABLE_NZ=0 CONFIG_PATH=$(basename $(dirname $0)) -python examples/start_rlvr_pipeline.py --config_path $CONFIG_PATH --config_name qwen3_8b_rlvr_deepspeed +python examples/start_rlvr_pipeline.py --config_path $CONFIG_PATH --config_name qwen3_30b_rlvr_fsdp2 From 05db7eb856093b7d510d5aad921fe8fada940978 Mon Sep 17 00:00:00 2001 From: UsernameFull Date: Tue, 23 Jun 2026 22:00:33 +0800 Subject: [PATCH 2/4] fix: stabilize cpu ci --- .github/workflows/ci-npu-test.yml | 13 ++-- tests/test_ref_worker_type_consistency.py | 82 ++++++++++++++++------- 2 files changed, 62 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci-npu-test.yml b/.github/workflows/ci-npu-test.yml index 6f6034fad..ee6411e36 100644 --- a/.github/workflows/ci-npu-test.yml +++ b/.github/workflows/ci-npu-test.yml @@ -40,6 +40,7 @@ jobs: cache: "pip" cache-dependency-path: | requirements_common.txt + requirements_vision.txt mcore_adapter/pyproject.toml mcore_adapter/requirements.txt setup.py @@ -50,13 +51,11 @@ jobs: pip install --upgrade pip # Install PyTorch CPU-only to keep CI lightweight pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu - # Install core test dependencies (subset of requirements_common.txt) - pip install pytest pytest-timeout pytest-asyncio numpy tensordict pydantic dacite \ - more_itertools hydra-core omegaconf peft==0.12.0 datasets==3.1.0 \ - trl==0.9.6 transformers ray[default] sympy deprecated codetiming pybase64 imageio \ - jsonschema mcp gem-llm==0.0.4 openai==2.31.0 gym 'gymnasium[toy-text]' gym_sokoban rl-rock - # Install mcore_adapter and roll itself - pip install -e ./mcore_adapter + # Install common dependencies, including mcore_adapter and TransferQueue + pip install -r requirements_common.txt + # Install CI-only pytest plugins and ROCK CLI used by agentic tests + pip install pytest-timeout pytest-asyncio rl-rock + # Install roll itself pip install -e . rock admin start diff --git a/tests/test_ref_worker_type_consistency.py b/tests/test_ref_worker_type_consistency.py index 6abddf7f0..d0a416fc7 100644 --- a/tests/test_ref_worker_type_consistency.py +++ b/tests/test_ref_worker_type_consistency.py @@ -18,40 +18,70 @@ def test_ref_worker_uses_cluster_not_config(): - """When use_ref_model=False, `worker` must be `self.actor_train` (Cluster), not `self.pipeline_config.actor_train` (WorkerConfig).""" + """When use_ref_model=False, log-prob computation must use `self.actor_train` (Cluster), not WorkerConfig.""" import roll.pipeline.rlvr.rlvr_pipeline as mod source = inspect.getsource(mod.RLVRPipeline) - - # The buggy pattern: `self.pipeline_config.actor_train` used where `self.actor_train` is needed - # The fix ensures `worker = ... else self.actor_train` (without pipeline_config prefix) - # - # We check: in the line that assigns `worker = ...`, the else-branch must NOT - # reference `self.pipeline_config.actor_train` tree = ast.parse(textwrap.dedent(source)) - found_worker_assign = False + def is_self_attr(node: ast.AST, *attrs: str) -> bool: + expected = ("self", *attrs) + current = node + actual = [] + while isinstance(current, ast.Attribute): + actual.append(current.attr) + current = current.value + if isinstance(current, ast.Name): + actual.append(current.id) + return tuple(reversed(actual)) == expected + + def is_not_self_use_ref_model(node: ast.AST) -> bool: + return ( + isinstance(node, ast.UnaryOp) + and isinstance(node.op, ast.Not) + and is_self_attr(node.operand, "use_ref_model") + ) + + lora_ref_branch = None for node in ast.walk(tree): - if not isinstance(node, ast.Assign): - continue - # Look for: worker = - for target in node.targets: - if isinstance(target, ast.Name) and target.id == "worker": - if isinstance(node.value, ast.IfExp): - found_worker_assign = True - # Check the orelse (else branch) of the ternary - orelse = node.value.orelse - # It should be self.actor_train, NOT self.pipeline_config.actor_train - source_segment = ast.dump(orelse) - assert "pipeline_config" not in source_segment, ( - "Bug: `worker` assignment else-branch references " - "`self.pipeline_config.actor_train` (WorkerConfig) instead of " - "`self.actor_train` (Cluster). WorkerConfig has no `dp_size` property." + if isinstance(node, ast.If) and is_not_self_use_ref_model(node.test): + lora_ref_branch = node + break + + assert lora_ref_branch is not None, ( + "Could not find the `if not self.use_ref_model` branch in RLVRPipeline. " + "The reference log-prob code structure may have changed." + ) + + found_actor_train_dp_size = False + found_actor_train_compute_log_probs = False + for node in ast.walk(lora_ref_branch): + if isinstance(node, ast.Attribute) and node.attr == "dp_size": + assert is_self_attr(node.value, "actor_train"), ( + "`dp_size` in the LoRA reference branch must come from " + "`self.actor_train` (Cluster), not WorkerConfig or another alias." + ) + found_actor_train_dp_size = True + + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute): + if node.func.attr == "compute_log_probs": + assert is_self_attr(node.func.value, "actor_train"), ( + "`compute_log_probs` in the LoRA reference branch must be called on " + "`self.actor_train` (Cluster), not WorkerConfig." + ) + found_actor_train_compute_log_probs = True + + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id == "worker": + assert "pipeline_config" not in ast.dump(node.value), ( + "Bug: `worker` is assigned from `self.pipeline_config.actor_train` " + "(WorkerConfig) instead of `self.actor_train` (Cluster)." ) - assert found_worker_assign, ( - "Could not find `worker = ... if ... else ...` ternary assignment in RLVRPipeline. " - "The code structure may have changed." + assert found_actor_train_dp_size, "LoRA reference branch should use `self.actor_train.dp_size`." + assert found_actor_train_compute_log_probs, ( + "LoRA reference branch should call `self.actor_train.compute_log_probs`." ) From ccaab5bdfb91f99406b914da98e665fd3054a2f9 Mon Sep 17 00:00:00 2001 From: UsernameFull Date: Wed, 24 Jun 2026 09:48:03 +0800 Subject: [PATCH 3/4] docs: rename Atlas A5 to 950 --- .../User Guides/Hardware Support/ascend_docker_usage.md | 8 ++++---- .../User Guides/Hardware Support/ascend_docker_usage.md | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs_roll/docs/User Guides/Hardware Support/ascend_docker_usage.md b/docs_roll/docs/User Guides/Hardware Support/ascend_docker_usage.md index e944aee50..5d08ee814 100644 --- a/docs_roll/docs/User Guides/Hardware Support/ascend_docker_usage.md +++ b/docs_roll/docs/User Guides/Hardware Support/ascend_docker_usage.md @@ -2,7 +2,7 @@ Last updated: 06/23/2026. -This guide explains how to get, build, and run ROLL images on **Huawei Ascend NPU**. Prefer the pre-built image when possible; use `Dockerfile.A2` or `Dockerfile.A3` when you need to customize dependencies. Atlas A5 currently follows the manual installation profile in [ROLL x Ascend](ascend_usage.md). +This guide explains how to get, build, and run ROLL images on **Huawei Ascend NPU**. Prefer the pre-built image when possible; use `Dockerfile.A2` or `Dockerfile.A3` when you need to customize dependencies. Atlas 950 currently follows the manual installation profile in [ROLL x Ascend](ascend_usage.md). ## Hardware & Software Requirements @@ -15,7 +15,7 @@ This guide explains how to get, build, and run ROLL images on **Huawei Ascend NP | Docker | >= 20.10 | >= 20.10 | | Ascend NPU Driver | Installed on host | Installed on host | -This Docker guide covers the A2/A3 Dockerfiles. For Atlas A5, use the manual installation profile: torch 2.10, vLLM v0.20.2, vLLM-Ascend `main`, and `COMPILE_CUSTOM_KERNELS=1` when building vLLM-Ascend. +This Docker guide covers the A2/A3 Dockerfiles. For Atlas 950, use the manual installation profile: torch 2.10, vLLM v0.20.2, vLLM-Ascend `main`, and `COMPILE_CUSTOM_KERNELS=1` when building vLLM-Ascend. ## Key Components @@ -29,9 +29,9 @@ Both Dockerfiles install the same versions of core dependencies: | Transformers | 4.57.6 | | triton-ascend | 3.2.1 | -Atlas A5 uses a newer manual installation stack: +Atlas 950 uses a newer manual installation stack: -| Component | Atlas A5 Version / Setting | +| Component | Atlas 950 Version / Setting | | --------- | -------------------------- | | PyTorch | 2.10 | | vLLM | v0.20.2 | diff --git a/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_docker_usage.md b/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_docker_usage.md index 6a7dc97d5..a83e7cb07 100644 --- a/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_docker_usage.md +++ b/docs_roll/i18n/zh-Hans/docusaurus-plugin-content-docs/current/User Guides/Hardware Support/ascend_docker_usage.md @@ -2,7 +2,7 @@ 最后更新：2026/06/23。 -本指南介绍如何在**华为昇腾 NPU** 上获取、构建并运行 ROLL 镜像。推荐优先使用预构建镜像；如需自定义依赖，再使用 `Dockerfile.A2` 或 `Dockerfile.A3` 构建。Atlas A5 当前使用 [ROLL x Ascend](ascend_usage.md) 中的手动安装配置。 +本指南介绍如何在**华为昇腾 NPU** 上获取、构建并运行 ROLL 镜像。推荐优先使用预构建镜像；如需自定义依赖，再使用 `Dockerfile.A2` 或 `Dockerfile.A3` 构建。Atlas 950 当前使用 [ROLL x Ascend](ascend_usage.md) 中的手动安装配置。 ## 硬件与软件要求 @@ -15,7 +15,7 @@ | Docker | >= 20.10 | >= 20.10 | | 昇腾 NPU 驱动 | 已安装在宿主机上 | 已安装在宿主机上 | -本 Docker 指南覆盖 A2/A3 Dockerfile。Atlas A5 请使用手动安装配置：torch 2.10、vLLM v0.20.2、vLLM-Ascend `main`，并在构建 vLLM-Ascend 时设置 `COMPILE_CUSTOM_KERNELS=1`。 +本 Docker 指南覆盖 A2/A3 Dockerfile。Atlas 950 请使用手动安装配置：torch 2.10、vLLM v0.20.2、vLLM-Ascend `main`，并在构建 vLLM-Ascend 时设置 `COMPILE_CUSTOM_KERNELS=1`。 ## 主要组件 @@ -29,9 +29,9 @@ | Transformers | 4.57.6 | | triton-ascend | 3.2.1 | -Atlas A5 使用更新的手动安装版本组合： +Atlas 950 使用更新的手动安装版本组合： -| 组件 | Atlas A5 版本 / 设置 | +| 组件 | Atlas 950 版本 / 设置 | | ---- | -------------------- | | PyTorch | 2.10 | | vLLM | v0.20.2 | From e72b4f422611184a07e29bc99de5946b7d569bfc Mon Sep 17 00:00:00 2001 From: UsernameFull Date: Fri, 26 Jun 2026 14:52:13 +0800 Subject: [PATCH 4/4] fix: add deepspeed dependency in ci --- .github/workflows/ci-npu-test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-npu-test.yml b/.github/workflows/ci-npu-test.yml index ee6411e36..df68b857d 100644 --- a/.github/workflows/ci-npu-test.yml +++ b/.github/workflows/ci-npu-test.yml @@ -197,7 +197,7 @@ jobs: - name: Install ROLL requirements run: | python3 -m pip install -r requirements_common.txt - python3 -m pip install tensorboard + python3 -m pip install deepspeed==0.16.4 tensorboard - name: Install SGLang NPU kernel from source shell: bash @@ -288,7 +288,7 @@ jobs: except metadata.PackageNotFoundError: return "not installed" - packages = ("vllm-ascend", "transformers", "triton-ascend") + packages = ("vllm-ascend", "transformers", "deepspeed", "triton-ascend") for package_name in packages: print(f"{package_name}={package_version(package_name)}") @@ -301,6 +301,7 @@ jobs: export PYTHONPATH="${GITHUB_WORKSPACE}:${PYTHONPATH:-}" python3 -m pytest tests/third_party/sglang \ tests/third_party/vllm \ + tests/third_party/deepspeed \ tests/distributed \ tests/models \ tests/pipeline \