Skip to content

[WIP] fix formers ci #76700

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: develop
Choose a base branch
from
+103 −48
Open

[WIP] fix formers ci #76700

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
468f883
fix formers ci
ooooo-create Nov 30, 2025
b24b85f
refactor
ooooo-create Nov 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 59 additions & 15 deletions .github/workflows/_Distribute-stable-Formers.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ jobs:
CCACHE_DIR: /root/.ccache/formers
CFS_DIR: /home/data/cfs
paddle_whl: /workspace/dist/*.whl
AISTUDIO_PROXY_PATH: ${{ github.workspace }}/../../../proxy_aistudio
formers_docker: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda126-dev-latest
run: |
export CUDA_SO="$(\ls -d /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls -d /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')"
Expand Down Expand Up @@ -125,6 +126,7 @@ jobs:
-e CFS_DIR \
-e paddle_whl \
-e no_proxy \
-e AISTUDIO_PROXY_PATH \
-w /workspace --network host ${formers_docker}
- name: Download paddle.tar.gz and merge target branch
run: |
Expand All @@ -138,16 +140,51 @@ jobs:
git config --global --add safe.directory /workspace
git checkout test
'
- name: Test
- name: Check PR Title
id: check_pr
run: |
if docker exec ${{ env.container_name }} /bin/bash -c 'source ${{ github.workspace }}/../../../proxy; python /workspace/tools/get_pr_title.py skip_distribute_test' > /dev/null 2>&1; then
echo "PR's title with 'CINN' or 'BUAA', skip the run formers ci test !"
echo "SKIP_FORMERS_TEST=true" >> $GITHUB_ENV
else
echo "SKIP_FORMERS_TEST=false" >> $GITHUB_ENV
fi

- name: Install Dependencies
id: install
if: env.SKIP_FORMERS_TEST != 'true'
run: |
docker exec -t ${{ env.container_name }} /bin/bash -c '
source ${{ github.workspace }}/../../../proxy
source ${{ github.workspace }}/../../../AISTUDIO_ACCESS_TOKEN
set -ex
bash /workspace/ci/formers_test.sh
bash /workspace/ci/formers_test.sh setup
'

- name: Run Formers API Tests
id: api_test
if: env.SKIP_FORMERS_TEST != 'true' && steps.install.outcome == 'success' && !cancelled()
run: |
docker exec -t ${{ env.container_name }} /bin/bash -c '
source ${{ github.workspace }}/../../../proxy
source ${{ github.workspace }}/../../../AISTUDIO_ACCESS_TOKEN
set -ex
bash /workspace/ci/formers_test.sh api
'

- name: Run Formers Models Tests
id: model_test
if: env.SKIP_FORMERS_TEST != 'true' && steps.install.outcome == 'success' && !cancelled()
run: |
docker exec -t ${{ env.container_name }} /bin/bash -c '
source ${{ github.workspace }}/../../../proxy
source ${{ github.workspace }}/../../../AISTUDIO_ACCESS_TOKEN
set -ex
bash /workspace/ci/formers_test.sh models
'

- name: Upload and display logs
if: always()
if: always() && env.SKIP_FORMERS_TEST != 'true' && steps.install.outcome == 'success' && !cancelled()
env:
home_path: ${{ github.workspace }}/..
bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
Expand All @@ -165,21 +202,28 @@ jobs:
else
bos_prefix="schedule/$(date +%Y%m%d)"
fi

# api test logs
cd /workspace/PaddleFormers/unittest_logs
for FILE in /workspace/PaddleFormers/unittest_logs/*; do
file=$(basename "$FILE")
python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleFormers/unittest-gpu/${bos_prefix}/logs
echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/unittest-gpu/${bos_prefix}/logs/$file"
done
if [ -d "/workspace/PaddleFormers/unittest_logs" ]; then
cd /workspace/PaddleFormers/unittest_logs
for FILE in /workspace/PaddleFormers/unittest_logs/*; do
file=$(basename "$FILE")
python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleFormers/unittest-gpu/${bos_prefix}/logs
echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/unittest-gpu/${bos_prefix}/logs/$file"
done
fi

# models test logs
cd /workspace/PaddleFormers/model_unittest_logs
for FILE in /workspace/PaddleFormers/model_unittest_logs/*; do
file=$(basename "$FILE")
python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs
echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs/$file"
done
if [ -d "/workspace/PaddleFormers/model_unittest_logs" ]; then
cd /workspace/PaddleFormers/model_unittest_logs
for FILE in /workspace/PaddleFormers/model_unittest_logs/*; do
file=$(basename "$FILE")
python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs
echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs/$file"
done
fi
'

- name: Terminate and delete the container
if: always()
run: |
Expand Down
77 changes: 44 additions & 33 deletions ci/formers_test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,57 +12,68 @@
# See the License for the specific language governing permissions and
# limitations under the License.

function init_env() {
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib64/:/usr/local/lib/
export PATH=/usr/local/bin:${PATH}
}

function formers_api() {
init_env
cd /workspace/PaddleFormers && git config --global --add safe.directory $PWD
echo "Check whether the local model file exists:"
ls -l ./models
sed -i '/python setup.py bdist_wheel > \/dev\/null/d' scripts/unit_test/ci_unittest.sh
timeout 30m bash scripts/unit_test/ci_unittest.sh ${paddle_whl} false ${PYTEST_EXECUTE_FLAG_FILE} ${BRANCH}
}

function formers_models() {
init_env
rm -rf /root/.cache/aistudio/
cd /workspace/PaddleFormers && git config --global --add safe.directory $PWD
echo "Check whether the local model file exists:"
ls -l ./models
sed -i '/python setup.py bdist_wheel > \/dev\/null/d' scripts/regression/ci_model_unittest.sh
timeout 30m bash scripts/regression/ci_model_unittest.sh ${paddle_whl} ${BRANCH}
}

function formers_test() {
python /workspace/tools/get_pr_title.py skip_distribute_test && CINN_OR_BUAA_PR=1
if [[ "${CINN_OR_BUAA_PR}" = "1" ]];then
echo "PR's title with 'CINN' or 'BUAA', skip the run distribute ci test !"
exit 0
fi
function install_deps() {
set -e
init_env
ln -sf $(which python3.10) /usr/local/bin/python
ln -sf $(which pip3.10) /usr/local/bin/pip

echo "::group::Start formers api tests"
formers_api
echo "End api tests"
echo "::endgroup::"
echo "Downloading PaddleFormers.tar.gz..."
wget -q https://paddleformers.bj.bcebos.com/wheels/PaddleFormers.tar.gz
tar xf PaddleFormers.tar.gz
echo "Extracting PaddleFormers.tar.gz..."

if [ -d "PaddleFormers" ]; then
cd PaddleFormers
cp -r ${CFS_DIR}/models ./models
else
echo "Error: PaddleFormers dir not found after tar xf"
exit 1
fi

echo "::group::Start formers models tests"
formers_models
echo "End models tests"
echo "::group::Install paddle dependencies"
pip config set global.cache-dir "/root/.cache/pip"
pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
echo "::endgroup::"
ldconfig
}

set -e
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib64/:/usr/local/lib/
PATH=/usr/local/bin:${PATH}
ln -sf $(which python3.10) /usr/local/bin/python
ln -sf $(which pip3.10) /usr/local/bin/pip

echo "Downloading PaddleFormers.tar.gz..."
wget -q https://paddleformers.bj.bcebos.com/wheels/PaddleFormers.tar.gz
tar xf PaddleFormers.tar.gz
echo "Extracting PaddleFormers.tar.gz..."
cd PaddleFormers
cp -r ${CFS_DIR}/models ./models

echo "::group::Install paddle dependencies"
pip config set global.cache-dir "/root/.cache/pip"
pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
echo "::endgroup::"
ldconfig

formers_test
case "$1" in
setup)
install_deps
;;
api)
formers_api
;;
models)
formers_models
;;
*)
echo "Usage: $0 {setup|api|models}"
exit 1
;;
esac
Loading