diff --git a/.github/workflows/_Distribute-stable-Formers.yml b/.github/workflows/_Distribute-stable-Formers.yml index acc053109e536d..e50e122c6db03b 100644 --- a/.github/workflows/_Distribute-stable-Formers.yml +++ b/.github/workflows/_Distribute-stable-Formers.yml @@ -73,6 +73,7 @@ jobs: CCACHE_DIR: /root/.ccache/formers CFS_DIR: /home/data/cfs paddle_whl: /workspace/dist/*.whl + AISTUDIO_PROXY_PATH: ${{ github.workspace }}/../../../proxy_aistudio formers_docker: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda126-dev-latest run: | export CUDA_SO="$(\ls -d /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls -d /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')" @@ -125,6 +126,7 @@ jobs: -e CFS_DIR \ -e paddle_whl \ -e no_proxy \ + -e AISTUDIO_PROXY_PATH \ -w /workspace --network host ${formers_docker} - name: Download paddle.tar.gz and merge target branch run: | @@ -138,16 +140,51 @@ jobs: git config --global --add safe.directory /workspace git checkout test ' - - name: Test + - name: Check PR Title + id: check_pr + run: | + if docker exec ${{ env.container_name }} /bin/bash -c 'source ${{ github.workspace }}/../../../proxy; python /workspace/tools/get_pr_title.py skip_distribute_test' > /dev/null 2>&1; then + echo "PR's title with 'CINN' or 'BUAA', skip the run formers ci test !" + echo "SKIP_FORMERS_TEST=true" >> $GITHUB_ENV + else + echo "SKIP_FORMERS_TEST=false" >> $GITHUB_ENV + fi + + - name: Install Dependencies + id: install + if: env.SKIP_FORMERS_TEST != 'true' run: | docker exec -t ${{ env.container_name }} /bin/bash -c ' source ${{ github.workspace }}/../../../proxy source ${{ github.workspace }}/../../../AISTUDIO_ACCESS_TOKEN set -ex - bash /workspace/ci/formers_test.sh + bash /workspace/ci/formers_test.sh setup ' + + - name: Run Formers API Tests + id: api_test + if: env.SKIP_FORMERS_TEST != 'true' && steps.install.outcome == 'success' && !cancelled() + run: | + docker exec -t ${{ env.container_name }} /bin/bash -c ' + source ${{ github.workspace }}/../../../proxy + source ${{ github.workspace }}/../../../AISTUDIO_ACCESS_TOKEN + set -ex + bash /workspace/ci/formers_test.sh api + ' + + - name: Run Formers Models Tests + id: model_test + if: env.SKIP_FORMERS_TEST != 'true' && steps.install.outcome == 'success' && !cancelled() + run: | + docker exec -t ${{ env.container_name }} /bin/bash -c ' + source ${{ github.workspace }}/../../../proxy + source ${{ github.workspace }}/../../../AISTUDIO_ACCESS_TOKEN + set -ex + bash /workspace/ci/formers_test.sh models + ' + - name: Upload and display logs - if: always() + if: always() && env.SKIP_FORMERS_TEST != 'true' && steps.install.outcome == 'success' && !cancelled() env: home_path: ${{ github.workspace }}/.. bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py @@ -165,21 +202,28 @@ jobs: else bos_prefix="schedule/$(date +%Y%m%d)" fi + # api test logs - cd /workspace/PaddleFormers/unittest_logs - for FILE in /workspace/PaddleFormers/unittest_logs/*; do - file=$(basename "$FILE") - python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleFormers/unittest-gpu/${bos_prefix}/logs - echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/unittest-gpu/${bos_prefix}/logs/$file" - done + if [ -d "/workspace/PaddleFormers/unittest_logs" ]; then + cd /workspace/PaddleFormers/unittest_logs + for FILE in /workspace/PaddleFormers/unittest_logs/*; do + file=$(basename "$FILE") + python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleFormers/unittest-gpu/${bos_prefix}/logs + echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/unittest-gpu/${bos_prefix}/logs/$file" + done + fi + # models test logs - cd /workspace/PaddleFormers/model_unittest_logs - for FILE in /workspace/PaddleFormers/model_unittest_logs/*; do - file=$(basename "$FILE") - python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs - echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs/$file" - done + if [ -d "/workspace/PaddleFormers/model_unittest_logs" ]; then + cd /workspace/PaddleFormers/model_unittest_logs + for FILE in /workspace/PaddleFormers/model_unittest_logs/*; do + file=$(basename "$FILE") + python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs + echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs/$file" + done + fi ' + - name: Terminate and delete the container if: always() run: | diff --git a/ci/formers_test.sh b/ci/formers_test.sh index 6e8f9c1271a876..c394345de3b884 100644 --- a/ci/formers_test.sh +++ b/ci/formers_test.sh @@ -12,57 +12,68 @@ # See the License for the specific language governing permissions and # limitations under the License. +function init_env() { + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib64/:/usr/local/lib/ + export PATH=/usr/local/bin:${PATH} +} function formers_api() { + init_env cd /workspace/PaddleFormers && git config --global --add safe.directory $PWD echo "Check whether the local model file exists:" ls -l ./models + sed -i '/python setup.py bdist_wheel > \/dev\/null/d' scripts/unit_test/ci_unittest.sh timeout 30m bash scripts/unit_test/ci_unittest.sh ${paddle_whl} false ${PYTEST_EXECUTE_FLAG_FILE} ${BRANCH} } function formers_models() { + init_env rm -rf /root/.cache/aistudio/ cd /workspace/PaddleFormers && git config --global --add safe.directory $PWD echo "Check whether the local model file exists:" ls -l ./models + sed -i '/python setup.py bdist_wheel > \/dev\/null/d' scripts/regression/ci_model_unittest.sh timeout 30m bash scripts/regression/ci_model_unittest.sh ${paddle_whl} ${BRANCH} } -function formers_test() { - python /workspace/tools/get_pr_title.py skip_distribute_test && CINN_OR_BUAA_PR=1 - if [[ "${CINN_OR_BUAA_PR}" = "1" ]];then - echo "PR's title with 'CINN' or 'BUAA', skip the run distribute ci test !" - exit 0 - fi +function install_deps() { + set -e + init_env + ln -sf $(which python3.10) /usr/local/bin/python + ln -sf $(which pip3.10) /usr/local/bin/pip - echo "::group::Start formers api tests" - formers_api - echo "End api tests" - echo "::endgroup::" + echo "Downloading PaddleFormers.tar.gz..." + wget -q https://paddleformers.bj.bcebos.com/wheels/PaddleFormers.tar.gz + tar xf PaddleFormers.tar.gz + echo "Extracting PaddleFormers.tar.gz..." + + if [ -d "PaddleFormers" ]; then + cd PaddleFormers + cp -r ${CFS_DIR}/models ./models + else + echo "Error: PaddleFormers dir not found after tar xf" + exit 1 + fi - echo "::group::Start formers models tests" - formers_models - echo "End models tests" + echo "::group::Install paddle dependencies" + pip config set global.cache-dir "/root/.cache/pip" + pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple echo "::endgroup::" + ldconfig } -set -e -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib64/:/usr/local/lib/ -PATH=/usr/local/bin:${PATH} -ln -sf $(which python3.10) /usr/local/bin/python -ln -sf $(which pip3.10) /usr/local/bin/pip - -echo "Downloading PaddleFormers.tar.gz..." -wget -q https://paddleformers.bj.bcebos.com/wheels/PaddleFormers.tar.gz -tar xf PaddleFormers.tar.gz -echo "Extracting PaddleFormers.tar.gz..." -cd PaddleFormers -cp -r ${CFS_DIR}/models ./models - -echo "::group::Install paddle dependencies" -pip config set global.cache-dir "/root/.cache/pip" -pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple -echo "::endgroup::" -ldconfig - -formers_test +case "$1" in + setup) + install_deps + ;; + api) + formers_api + ;; + models) + formers_models + ;; + *) + echo "Usage: $0 {setup|api|models}" + exit 1 + ;; +esac