diff --git a/.github/workflows/Publish_IXUCA.yaml b/.github/workflows/Publish_IXUCA.yaml new file mode 100644 index 0000000000..922dac6060 --- /dev/null +++ b/.github/workflows/Publish_IXUCA.yaml @@ -0,0 +1,192 @@ +name: paddle ixuca build and publish + +on: + workflow_dispatch: + schedule: + - cron: '0 18 * * *' # 2:00 AM China Standard Time (UTC+8) + push: + tags: + - '*' + pull_request: + types: [opened, synchronize] + branches: [develop, release/**] + +permissions: read-all + +defaults: + run: + shell: bash + + +jobs: + code-checkout: + runs-on: + group: HK-Clone + env: + BRANCH: ${{ github.ref_name }} + COMMIT: ${{ github.sha }} + outputs: + repo_archive_url: ${{ steps.code_upload.outputs.repo_archive_url }} + steps: + - name: Clone PaddleCustomDevice + uses: actions/checkout@v4 + with: + ref: ${{ github.ref_name }} + submodules: 'recursive' + fetch-depth: 1000 + - name: Download bos client + env: + home_path: "/home/paddle/actions-runner/" + bos_file: "/home/paddle/actions-runner/bos/BosClient.py" + run: | + if [ ! -f "${bos_file}" ]; then + wget -q --no-proxy -O ${home_path}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate + mkdir ${home_path}/bos + tar xf ${home_path}/bos_new.tar.gz -C ${home_path}/bos + fi + - name: Push paddle-action.tar.gz to bos + env: + AK: paddle + SK: paddle + bos_file: "/home/paddle/actions-runner/bos/BosClient.py" + id: code_upload + run: | + set -x + git reset --hard ${COMMIT} + git log -n 3 + cd .. + tar -I 'zstd -T0' -cf PaddleCustomDevice.tar.gz PaddleCustomDevice + echo "::group::Install bce-python-sdk" + python -m pip install bce-python-sdk==0.8.74 + echo "::endgroup::" + python ${bos_file} PaddleCustomDevice.tar.gz paddle-github-action/PaddleCustomDevice/COMMIT/${BRANCH}/${COMMIT} + rm PaddleCustomDevice.tar.gz + repo_archive_url=https://paddle-github-action.bj.bcebos.com/PaddleCustomDevice/COMMIT/${BRANCH}/${COMMIT}/PaddleCustomDevice.tar.gz + echo "repo_archive_url=${repo_archive_url}" >> $GITHUB_OUTPUT + + ixuca-gpu-publish: + needs: code-checkout + environment: PublishUploadUse + env: + PaddleCustomDeviceVersion: ${{ vars.PaddleCustomDeviceVersion }} + AK: ${{ secrets.BOS_AK }} + SK: ${{ secrets.BOS_SK }} + TASK: Publish-${{ github.sha }}-ixuca + no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn" + REPO_ARCHIVE_URL: ${{ needs.code-checkout.outputs.repo_archive_url }} + BRANCH: ${{ github.ref_name }} + COMMIT: ${{ github.sha }} + REF_TYPE: ${{ github.ref_type }} + runs-on: + group: test2 + steps: + - name: Cleanup + run: | + rm -rf * .[^.]* + - name: Check docker image and run container + env: + GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_ID: ${{ github.run_id }} + cache_dir: /root/.cache + ccache_dir: /root/.ccache + docker_image: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/device/paddle-ixuca:latest + run: | + container_name=${TASK}-$(date +%Y%m%d-%H%M%S) + echo "container_name=${container_name}" >> $GITHUB_ENV + docker run --shm-size=128G \ + -d -t --name ${container_name} \ + --privileged --network=host \ + -e PR_ID \ + -e COMMIT_ID \ + -e BRANCH \ + -e no_proxy \ + -e PaddleCustomDeviceVersion \ + -e AK \ + -e SK \ + -e LD_LIBRARY_PATH=/usr/local/corex-4.3.0/lib \ + -e LIBRARY_PATH=/usr/local/corex-4.3.0/lib \ + -v ${cache_dir}:/root/.cache \ + -v ${ccache_dir}:/root/.ccache \ + -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \ + -v ${{ github.workspace }}:/workspace -w /workspace \ + ${docker_image} /bin/bash + + - name: Download Code + run: | + docker exec -t ${{ env.container_name }} /bin/bash -c ' + source ${{ github.workspace }}/../../../proxy + set -x + wget -q --tries=5 --no-proxy ${REPO_ARCHIVE_URL} --no-check-certificate + echo "Extracting PaddleCustomDevice.tar.gz" + tar -xf PaddleCustomDevice.tar.gz + cd PaddleCustomDevice + git config --global --add safe.directory "*" + git --no-pager log --pretty=oneline -5' + + - name: RUN IXUCA + id: run-ixuca + run: | + docker exec -t ${{ env.container_name }} /bin/bash -c ' + cd /workspace/PaddleCustomDevice + export PATH=/usr/local/corex-4.3.0/bin:$PATH + mkdir -p /workspace/PaddleCustomDevice/Paddle/third_party/mklml/Linux + curl -o /workspace/PaddleCustomDevice/Paddle/third_party/mklml/Linux/csrmm_mklml_lnx_2019.0.5.tgz http://paddlepaddledeps.bj.bcebos.com/csrmm_mklml_lnx_2019.0.5.tgz + tar xf /workspace/PaddleCustomDevice/Paddle/third_party/mklml/Linux/csrmm_mklml_lnx_2019.0.5.tgz + python3 -m pip install --pre paddlepaddle -i https://www.paddlepaddle.org.cn/packages/nightly/cpu/ + python3 -m pip install parameterized' + + - name: Build and Install paddle_iluvatar_gpu + run: | + docker exec -t ${{ env.container_name }} /bin/bash -c ' + cd /workspace/PaddleCustomDevice/ + if [[ "${BRANCH}" == "develop" ]];then + GIT_COMMIT_TIME=$(git --no-pager show -s --format=%ci HEAD) + DATE_ONLY=$(echo "$GIT_COMMIT_TIME" | sed "s/ .*//;s/-//g") + echo "Git Commit Time: $GIT_COMMIT_TIME" + echo "Date Only: $DATE_ONLY" + export PADDLE_VERSION=${PaddleCustomDeviceVersion}.dev${DATE_ONLY} + export PLUGIN_VERSION=${PaddleCustomDeviceVersion}.dev${DATE_ONLY} + elif [[ "${REF_TYPE}" == "tag" ]]; then + VERSION="${BRANCH#v}" + echo "VERSION=$VERSION" + export PADDLE_VERSION="${VERSION}" + export PLUGIN_VERSION="${VERSION}" + else + echo "Not develop or tag, do nothing" + fi + cd /workspace/PaddleCustomDevice/backends/iluvatar_gpu + # build + bash build_paddle.sh + ' + + - name: push whl + env: + AK: ${{ secrets.BOS_AK }} + SK: ${{ secrets.BOS_SK }} + run: | + docker exec -t ${{ env.container_name }} /bin/bash -c ' + if [[ "${BRANCH}" == "develop" ]];then + target_path="paddle-whl/nightly/ixuca/paddle-iluvatar-gpu" + elif [[ "${REF_TYPE}" == "tag" ]]; then + target_path="paddle-whl/stable/ixuca/paddle-iluvatar-gpu" + else + echo "Not develop or tag, do nothing" + fi + pip install bce-python-sdk==0.8.74 + + if [ ! -f "BosClient.py}" ]; then + wget -q --no-proxy https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate + tar xf bos_retry.tar.gz + fi + cp backends/iluvatar_gpu/build/dist/paddle_iluvatar_gpu*.whl . + python BosClient.py paddle_iluvatar_gpu*.whl ${target_path} + ' + + - name: Terminate and delete the container + if: always() + run: | + docker exec -t ${{ env.container_name }} /bin/bash -c ' + rm -rf * .[^.]* + ' + docker stop ${container_name} + docker rm ${container_name} diff --git a/.github/workflows/Publish_Metax.yaml b/.github/workflows/Publish_Metax.yaml index a3e6b417a0..b2392e00fb 100644 --- a/.github/workflows/Publish_Metax.yaml +++ b/.github/workflows/Publish_Metax.yaml @@ -17,6 +17,8 @@ defaults: jobs: metax-gpu-publish: environment: PublishUploadUse + env: + PaddleCustomDeviceVersion: ${{ vars.PaddleCustomDeviceVersion }} runs-on: paddle-metax-runner-set steps: - name: Checkout repository