Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 6 additions & 8 deletions .github/workflows/ci-npu-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ jobs:
cache: "pip"
cache-dependency-path: |
requirements_common.txt
requirements_vision.txt
mcore_adapter/pyproject.toml
mcore_adapter/requirements.txt
setup.py
Expand All @@ -50,13 +51,11 @@ jobs:
pip install --upgrade pip
# Install PyTorch CPU-only to keep CI lightweight
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
# Install core test dependencies (subset of requirements_common.txt)
pip install pytest pytest-timeout pytest-asyncio numpy tensordict pydantic dacite \
more_itertools hydra-core omegaconf peft==0.12.0 datasets==3.1.0 \
trl==0.9.6 transformers ray[default] sympy deprecated codetiming pybase64 imageio \
jsonschema mcp gem-llm==0.0.4 openai==2.31.0 gym 'gymnasium[toy-text]' gym_sokoban rl-rock
# Install mcore_adapter and roll itself
pip install -e ./mcore_adapter
# Install common dependencies, including mcore_adapter and TransferQueue
pip install -r requirements_common.txt
# Install CI-only pytest plugins and ROCK CLI used by agentic tests
pip install pytest-timeout pytest-asyncio rl-rock
# Install roll itself
pip install -e .
rock admin start

Expand Down Expand Up @@ -315,4 +314,3 @@ jobs:
-v --timeout=600 --durations=0 --durations-min=0 -x
env:
ROLL_NPU_CI: "1"
DS_UNITTEST_TIMEOUT: "600"
18 changes: 10 additions & 8 deletions docker/Dockerfile.A2
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM quay.io/ascend/cann:8.5.1-910b-ubuntu22.04-py3.11
FROM quay.io/ascend/cann:9.0.0-910b-ubuntu22.04-py3.11

ARG SOC_VERSION="ascend910b1"

Expand All @@ -9,6 +9,7 @@ ENV PIP_CONSTRAINT=""

ENV LD_LIBRARY_PATH=/usr/local/Ascend/driver/lib64:/usr/local/Ascend/driver/lib64/common:/usr/local/Ascend/driver/lib64/driver:/usr/local/Ascend/add-ons:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64:/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64:/usr/local/Ascend/ascend-toolkit/latest/atc/lib64:${LD_LIBRARY_PATH}
ENV ASCEND_HOME_PATH=/usr/local/Ascend/ascend-toolkit/latest
ENV HCCL_NPU_SOCKET_PORT_RANGE=auto

RUN apt-get update -y && \
apt-get install -y --no-install-recommends \
Expand All @@ -26,8 +27,8 @@ RUN pip config set global.index-url https://mirrors.huaweicloud.com/repository/p

WORKDIR /workspace

RUN git clone --depth 1 -b v0.13.0 https://github.com/vllm-project/vllm.git && \
git clone --depth 1 -b releases/v0.13.0 https://github.com/vllm-project/vllm-ascend.git
RUN git clone --depth 1 -b v0.18.0 https://github.com/vllm-project/vllm.git && \
git clone --depth 1 -b v0.18.0 https://github.com/vllm-project/vllm-ascend.git

RUN cd vllm && \
pip install -r requirements/build.txt && \
Expand All @@ -36,7 +37,7 @@ RUN cd vllm && \
pip cache purge && \
cd ..

RUN pip install torch==2.8.0+cpu torchvision==0.23.0 torchaudio==2.8.0 \
RUN pip install torch==2.9.0+cpu torchvision==0.24.0 torchaudio==2.9.0 \
--index-url https://download.pytorch.org/whl/cpu

RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
Expand All @@ -50,17 +51,18 @@ COPY . .

RUN pip install -r requirements_common.txt

RUN pip install "deepspeed==0.16.4" "transformers==4.57.6" "tensorboard==2.20.0"
Comment thread
UsernameFull marked this conversation as resolved.
RUN pip install "transformers==4.57.6" "tensorboard==2.20.0"

RUN pip install -e .

RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
pip uninstall -y triton && \
pip uninstall -y triton-ascend && \
pip install triton-ascend==3.2.0 && \
pip install triton-ascend==3.2.1 --extra-index-url https://mirrors.huaweicloud.com/ascend/repos/pypi && \
pip cache purge

RUN echo "source /usr/local/Ascend/ascend-toolkit/set_env.sh" >> /root/.bashrc && \
echo "source /usr/local/Ascend/nnal/atb/set_env.sh" >> /root/.bashrc
echo "source /usr/local/Ascend/nnal/atb/set_env.sh" >> /root/.bashrc && \
echo "export HCCL_NPU_SOCKET_PORT_RANGE=auto" >> /root/.bashrc

CMD ["/bin/bash"]
CMD ["/bin/bash"]
18 changes: 10 additions & 8 deletions docker/Dockerfile.A3
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM quay.io/ascend/cann:8.5.1-a3-ubuntu22.04-py3.11
FROM quay.io/ascend/cann:9.0.0-a3-ubuntu22.04-py3.11

ARG SOC_VERSION="ascend910_9391"

Expand All @@ -9,6 +9,7 @@ ENV PIP_CONSTRAINT=""

ENV LD_LIBRARY_PATH=/usr/local/Ascend/driver/lib64:/usr/local/Ascend/driver/lib64/common:/usr/local/Ascend/driver/lib64/driver:/usr/local/Ascend/add-ons:/usr/local/Ascend/ascend-toolkit/latest/fwkacllib/lib64:/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64:/usr/local/Ascend/ascend-toolkit/latest/atc/lib64:${LD_LIBRARY_PATH}
ENV ASCEND_HOME_PATH=/usr/local/Ascend/ascend-toolkit/latest
ENV HCCL_NPU_SOCKET_PORT_RANGE=auto

RUN apt-get update -y && \
apt-get install -y --no-install-recommends \
Expand All @@ -26,8 +27,8 @@ RUN pip config set global.index-url https://mirrors.huaweicloud.com/repository/p

WORKDIR /workspace

RUN git clone --depth 1 -b v0.13.0 https://github.com/vllm-project/vllm.git && \
git clone --depth 1 -b releases/v0.13.0 https://github.com/vllm-project/vllm-ascend.git
RUN git clone --depth 1 -b v0.18.0 https://github.com/vllm-project/vllm.git && \
git clone --depth 1 -b v0.18.0 https://github.com/vllm-project/vllm-ascend.git

RUN cd vllm && \
pip install -r requirements/build.txt && \
Expand All @@ -36,7 +37,7 @@ RUN cd vllm && \
pip cache purge && \
cd ..

RUN pip install torch==2.8.0+cpu torchvision==0.23.0 torchaudio==2.8.0 \
RUN pip install torch==2.9.0+cpu torchvision==0.24.0 torchaudio==2.9.0 \
--index-url https://download.pytorch.org/whl/cpu

RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
Expand All @@ -50,17 +51,18 @@ COPY . .

RUN pip install -r requirements_common.txt

RUN pip install "deepspeed==0.16.4" "transformers==4.57.6" "tensorboard==2.20.0"
RUN pip install "transformers==4.57.6" "tensorboard==2.20.0"

RUN pip install -e .

RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
pip uninstall -y triton && \
pip uninstall -y triton-ascend && \
pip install triton-ascend==3.2.0 && \
pip install triton-ascend==3.2.1 --extra-index-url https://mirrors.huaweicloud.com/ascend/repos/pypi && \
pip cache purge

RUN echo "source /usr/local/Ascend/ascend-toolkit/set_env.sh" >> /root/.bashrc && \
echo "source /usr/local/Ascend/nnal/atb/set_env.sh" >> /root/.bashrc
echo "source /usr/local/Ascend/nnal/atb/set_env.sh" >> /root/.bashrc && \
echo "export HCCL_NPU_SOCKET_PORT_RANGE=auto" >> /root/.bashrc

CMD ["/bin/bash"]
CMD ["/bin/bash"]
103 changes: 88 additions & 15 deletions docs_roll/docs/User Guides/Hardware Support/ascend_docker_usage.md
Original file line number Diff line number Diff line change
@@ -1,41 +1,73 @@
# Running ROLL on Ascend NPU with Docker

Last updated: 04/27/2026.
Last updated: 06/23/2026.

This guide explains how to build and run ROLL on **Huawei Ascend NPU** using `Dockerfile.A2` and `Dockerfile.A3`.
This guide explains how to get, build, and run ROLL images on **Huawei Ascend NPU**. Prefer the pre-built image when possible; use `Dockerfile.A2` or `Dockerfile.A3` when you need to customize dependencies. Atlas 950 currently follows the manual installation profile in [ROLL x Ascend](ascend_usage.md).

## Hardware & Software Requirements

| Item | Dockerfile.A2 | Dockerfile.A3 |
| ---- | ------------- | ------------- |
| Hardware | Atlas 900 A2 PODc (Ascend 910B1) | Atlas 900 A3 PODc (Ascend 910_9391) |
| Host OS | Ubuntu 22.04 | Ubuntu 22.04 |
| CANN | 8.5.1 | 8.5.1 |
| CANN | 9.0.0 | 9.0.0 |
| Python | 3.11 | 3.11 |
| Docker | >= 20.10 | >= 20.10 |
| Ascend NPU Driver | Installed on host | Installed on host |

This Docker guide covers the A2/A3 Dockerfiles. For Atlas 950, use the manual installation profile: torch 2.10, vLLM v0.20.2, vLLM-Ascend `main`, and `COMPILE_CUSTOM_KERNELS=1` when building vLLM-Ascend.

## Key Components

Both Dockerfiles install the same versions of core dependencies:

| Component | Version |
| --------- | ------- |
| PyTorch | 2.8.0+cpu |
| vLLM | 0.13.0 |
| vLLM-Ascend | 0.13.0 |
| DeepSpeed | 0.16.4 |
| PyTorch | 2.9.0+cpu |
| vLLM | 0.18.0 |
| vLLM-Ascend | 0.18 |
| Transformers | 4.57.6 |
| triton-ascend | 3.2.0 |
| triton-ascend | 3.2.1 |

Atlas 950 uses a newer manual installation stack:

| Component | Atlas 950 Version / Setting |
| --------- | -------------------------- |
| PyTorch | 2.10 |
| vLLM | v0.20.2 |
| vLLM-Ascend | `main` branch |
| Required build variable | `COMPILE_CUSTOM_KERNELS=1` |

The primary difference is the base image and SOC version:

| Item | Dockerfile.A2 | Dockerfile.A3 |
| ---- | ------------- | ------------- |
| Base Image | `quay.io/ascend/cann:8.5.1-910b-ubuntu22.04-py3.11` | `quay.io/ascend/cann:8.5.1-a3-ubuntu22.04-py3.11` |
| Base Image | `quay.io/ascend/cann:9.0.0-910b-ubuntu22.04-py3.11` | `quay.io/ascend/cann:9.0.0-a3-ubuntu22.04-py3.11` |
| SOC_VERSION | `ascend910b1` | `ascend910_9391` |

## Build the Docker Image
## Get the Docker Image

### Option A: Use the Pre-built Image (Recommended)

Pull the image that matches your hardware, then tag it with the local name used by the commands below:

**For Atlas 900 A2 PODc (Ascend 910B1):**

```bash
docker pull quay.io/ascend/roll:main-a2
docker tag quay.io/ascend/roll:main-a2 roll:ascend-a2
```

**For Atlas 900 A3 PODc (Ascend 910_9391):**

```bash
docker pull quay.io/ascend/roll:main-a3
docker tag quay.io/ascend/roll:main-a3 roll:ascend-a3
```

Check https://quay.io/repository/ascend/roll?tab=tags for available image tags. If you use a pre-built image, continue with [Run the Container](#run-the-container).

### Option B: Build from Dockerfile

### 1. Clone the ROLL Repository

Expand Down Expand Up @@ -88,6 +120,7 @@ docker run -dit \
--device /dev/davinci4 \
--device /dev/davinci5 \
--device /dev/davinci6 \
--device /dev/davinci7 \
--device /dev/davinci_manager \
--device /dev/devmm_svm \
--device /dev/hisi_hdc \
Expand Down Expand Up @@ -115,6 +148,7 @@ docker run -dit \
--device /dev/davinci4 \
--device /dev/davinci5 \
--device /dev/davinci6 \
--device /dev/davinci7 \
--device /dev/davinci_manager \
--device /dev/devmm_svm \
--device /dev/hisi_hdc \
Expand All @@ -130,6 +164,44 @@ docker run -dit \
/bin/bash
```

### Multi-NPU Startup (Recommended for Training)

For multi-NPU training, mount all available NPU devices. Adjust the number of `--device /dev/davinciX` entries according to the NPU count on your node:

```bash
docker run -dit \
--name roll_ascend \
--device /dev/davinci0 \
--device /dev/davinci1 \
--device /dev/davinci2 \
--device /dev/davinci3 \
--device /dev/davinci4 \
--device /dev/davinci5 \
--device /dev/davinci6 \
--device /dev/davinci7 \
--device /dev/davinci_manager \
--device /dev/devmm_svm \
--device /dev/hisi_hdc \
-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
-v /usr/local/Ascend/add-ons:/usr/local/Ascend/add-ons \
-v /usr/local/dcmi:/usr/local/dcmi \
-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
-v /etc/ascend_install.info:/etc/ascend_install.info \
-v /home/$USER:/home/$USER \
-v /path/to/models:/path/to/models \
-v /path/to/data:/path/to/data \
--ipc=host \
--net=host \
roll:ascend-a3 \
/bin/bash
```

> **Note:**
> - `--device /dev/davinciX`: Mounts NPU devices. Add or remove entries based on available NPU count.
> - `--device /dev/davinci_manager`, `--device /dev/devmm_svm`, `--device /dev/hisi_hdc`: Required management devices for Ascend NPU.
> - `-v /usr/local/Ascend/driver`: Mounts the host Ascend driver.
> - `-v /path/to/models` and `-v /path/to/data`: Mount model weights and training data directories as needed.

### Enter the Container

```bash
Expand All @@ -152,7 +224,7 @@ npu-smi info
env | grep -E "ASCEND|LD_LIBRARY_PATH|PATH"

# Verify Python packages
python -c "import torch; import torch_npu; print(torch_npu.npu.is_available())"
python -c "import torch; import torch_npu; print(torch.npu.is_available())"
python -c "import vllm; print(f'vllm: {vllm.__version__}')"
python -c "import vllm_ascend; print(f'vllm_ascend available')"
```
Expand All @@ -161,20 +233,21 @@ python -c "import vllm_ascend; print(f'vllm_ascend available')"

### Important Configuration Notes

Since Megatron-LM training is not yet supported on Ascend NPU, you need to use **DeepSpeed** as the training backend. Make sure your configuration files use the following settings:
Since Megatron-LM is not supported on Ascend NPU, you need to use **FSDP2** as the training backend. Make sure your configuration files use the following settings:

1. Set `strategy_args` to use DeepSpeed
2. Set `device_mapping` to ensure training and inference are performed on different NPUs
1. Set `strategy_args` to use FSDP2

### Example: RLVR Pipeline

```bash
# After modifying model paths and adjusting device_mapping
python examples/start_rlvr_pipeline.py \
--config_path ascend_examples \
--config_name qwen3_8b_rlvr_deepspeed
--config_name qwen3_30b_rlvr_fsdp2
```

> **Note:** The `qwen3_30b_rlvr_fsdp2` configuration is specifically designed for Ascend NPU with FSDP2 as the training backend. Adjust `device_mapping` in the configuration file according to your NPU topology.

## Troubleshooting

### NPU Not Visible Inside Container
Expand Down
Loading
Loading