.github/workflows/build-deploy-linux-cuda-11.4.yml
on:
workflow_dispatch:
inputs:
buildThreads:
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
required: true
default: 4
deployToReleaseStaging:
description: 'Whether to deploy to release staging or not.'
required: false
default: 0
releaseVersion:
description: 'Release version target'
required: false
default: 1.0.0-M3
snapshotVersion:
description: 'Snapshot version target'
required: false
default: 1.0.0-SNAPSHOT
releaseRepoId:
description: 'Release repository id'
required: false
default:
serverId:
description: 'Server id to publish to'
required: false
default: ossrh
mvnFlags:
description: "Extra maven flags (must escape input yourself if used)"
required: false
default:
libnd4jUrl:
description: 'Whether to download libnd4j using https://github.com/KonduitAI/gh-actions-libnd4j-urls/ for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
required: false
default:
runsOn:
description: 'System to run on'
required: false
default: ubuntu-20.04
cudaVersion:
description: 'Pick the cuda version'
required: false
default: 11.4
jobs:
linux-x86_64-cuda:
strategy:
fail-fast: false
matrix:
helper: [ cudnn, "" ]
extension: [ "" ]
include:
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
experimental: true
name: Extra maven flags
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
experimental: true
name: Debug enabled
- runs_on: ${{ github.event.inputs.runsOn }}
experimental: true
name: OS to run on
- libnd4j_file_download: ${{ github.event.inputs.libnd4jUrl }}
experimental: true
name: OS to run on
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
experimental: true
name: Whether to deploy to release staging or not
- release_version: ${{ github.event.inputs.releaseVersion }}
experimental: true
name: Release version
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
experimental: true
name: Snapshot version
- server_id: ${{ github.event.inputs.serverId }}
experimental: true
name: Server id
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
experimental: true
name: The release repository to run on
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
experimental: true
name: Extra maven flags to use as part of the build
- build_threads: ${{ github.event.inputs.buildThreads }}
experimental: true
name: The number of threads to build libnd4j with
- cuda_version: ${{ github.event.inputs.cudaVersion }}
experimental: true
name: CUDA version
runs-on: ${{ matrix.runs_on }}
steps:
- name: Cancel Previous Runs
uses: styfle/cancel-workflow-action@0.8.0
with:
access_token: ${{ github.token }}
- uses: actions/checkout@v2
- uses: ./.github/actions/set-linux-distro-version
- name: Set mvn build command based on matrix
shell: bash
run: |
if [ "${{ matrix.libnd4j_file_download }}" != '' ]; then
modules=':nd4j-cuda-${{ matrix.cuda_version }},:nd4j-cuda-${{ matrix.cuda_version }}-preset,:nd4j-cuda-${{ matrix.cuda_version }}-platform'
elif [ "${{ matrix.helper }}" == '' ]; then
echo "Building libnd4j from source"
modules=':nd4j-cuda-${{ matrix.cuda_version }},:nd4j-cuda-${{ matrix.cuda_version }}-preset,:libnd4j,:nd4j-cuda-${{ matrix.cuda_version }}-platform'
else
echo "Building libnd4j from source"
modules=':nd4j-cuda-${{ matrix.cuda_version }},:nd4j-cuda-${{ matrix.cuda_version }}-preset,:libnd4j'
fi
command="mvn ${{ matrix.mvn_ext }} -Pcuda --also-make -pl ${modules} -Dlibnd4j.compute='3.7 5.0 5.2 6.0 6.1 7.0 7.5 8.0 8.6' -Dlibnd4j.cpu.compile.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -Dlibnd4j.buildthreads=${{ matrix.build_threads }} -Djavacpp.platform=linux-x86_64 -Dlibnd4j.chip=cuda -Pcuda clean --batch-mode package deploy -DskipTests"
libnd4j_download_file_url=""
if [ "${{ matrix.helper }}" != '' ] && [ "${{ matrix.extension }}" != '' ]; then
mvn_ext= "-Pcuda -Djavacpp.platform.extension=-${{ matrix.helper }}-${{ matrix.extension }} -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.extension }} -Dlibnd4j.classifier=linux-x86_64-cuda-${{ matrix.cuda_version }}-${{ matrix.helper }}-${{matrix.extension}}"
libnd4j_download_file_url="linux-cuda-${{ matrix.cuda_version }}-${{ matrix.extension }}-${{ matrix.helper }}"
elif [ "${{ matrix.helper }}" != '' ]; then
mvn_ext=" -Djavacpp.platform.extension=-${{ matrix.helper }} -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.classifier=linux-x86_64-cuda-${{ matrix.cuda_version }}-${{ matrix.helper }}"
libnd4j_download_file_url="linux-cuda-${{ matrix.cuda_version }}-${{ matrix.helper }}"
else
libnd4j_download_file_url="linux-cuda-11"
mvn_ext=" -Pcuda -Dlibnd4j.classifier=linux-x86_64-cuda-${{ matrix.cuda_version }}"
fi
command="${command} ${mvn_ext}"
echo "Setting command for helper ${{ matrix.helper }} and extension ${{ matrix.extension }} to ${command} and libnd4j download file to ${libnd4j_download_file_url}"
echo "COMMAND=${command}" >> $GITHUB_ENV
- uses: ./.github/actions/update-deps-linux
- uses: ./.github/actions/remove-unneeded-tools-linux
- name: Cache cmake install
uses: actions/cache@v2
id: cache-cmake
with:
path: /opt/cmake
key: ${{ matrix.runs_on }}-cmake
restore-keys: ${{ matrix.runs_on }}-cmake
- name: Cache protobuf install
uses: actions/cache@v2
id: cache-protobuf
with:
path: /opt/protobuf
key: ${{ matrix.runs_on }}-protobuf
restore-keys: ${{ matrix.runs_on }}-protobuf
- uses: ./.github/actions/install-protobuf-linux
if: steps.cache-protobuf.outputs.cache-hit != 'true'
- uses: ./.github/actions/install-cmake-linux
if: steps.cache-cmake.outputs.cache-hit != 'true'
- name: Set up Java for publishing to GitHub Packages
uses: actions/setup-java@v2
with:
java-version: 11
distribution: 'temurin'
server-id: ${{ github.event.inputs.serverId }}
server-username: MAVEN_USERNAME
server-password: MAVEN_PASSWORD
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
gpg-passphrase: MAVEN_GPG_PASSPHRASE
cache: 'maven'
- run: |
sudo apt list --installed
- uses: konduitai/cuda-install/.github/actions/install-cuda-ubuntu@master
env:
cuda: ${{ matrix.cuda_version }}
GCC: 7
if: steps.cache-cuda-114.outputs.cache-hit != 'true'
- name: Build cuda
shell: bash
env:
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
DEBIAN_FRONTEND: noninteractive
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PUBLISH_TO: ossrh
MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }}
MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }}
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
PERFORM_RELEASE: ${{ matrix.deploy_to_release_staging }}
RELEASE_VERSION: ${{ matrix.release_version }}
SNAPSHOT_VERSION: ${{ matrix.snapshot_version }}
RELEASE_REPO_ID: ${{ matrix.release_repo_id }}
MODULES: ${{ matrix.mvn_flags }}
CUDA_PATH: /usr/local/cuda-${{ matrix.cuda_version }}
CUDNN_ROOT_DIR: /usr/local/cuda-${{ matrix.cuda_version }}/
LIBND4J_HOME_SUFFIX: cuda
HELPER: ${{ matrix.helper }}
EXTENSION: ${{ matrix.extension }}
LIBND4J_FILE_NAME: ${{ matrix.libnd4j_file_download }}
run: |
echo "libnd4j build threads ${{ matrix.build_threads }}"
echo "deploy to release staging repo or not ${{ matrix.deploy_to_release_staging }}"
echo "release version ${{ matrix.release_version }}"
echo "snapshot version ${{ matrix.snapshot_version }}"
echo "cuda version ${{ matrix.cuda_version }}"
echo "libnd4j url ${{ matrix.libnd4j_file_download }}"
echo "maven flags ${{ matrix.mvn_flags }}"
echo "snapshot version ${{ matrix.snapshot_version }}"
echo "server id ${{ matrix.server_id }}"
echo "release repo id ${{ matrix.release_repo_id }}"
bash ./change-cuda-versions.sh ${{ matrix.cuda_version }}
sudo apt list --installed
export PATH=/opt/protobuf/bin:/opt/cmake/bin:$PATH
export CUDA_PATH=/usr/local/cuda-${{ matrix.cuda_version }}
export PATH="/usr/local/cuda-${{ matrix.cuda_version }}/bin:$PATH"
# Note: we need this for the cudnn helpers, our cmake can't find it otherwise.
# See here: https://github.com/eclipse/deeplearning4j/blob/master/libnd4j/CMakeLists.txt#L298
echo "Helper is ${HELPER}"
if [ "${HELPER}" == "cudnn" ]; then
echo "Linking against cudnn"
echo "CUDA PATH is ${CUDA_PATH} CUDA_ROOT_DIR ${CUDNN_ROOT_DIR}"
ls -R "${CUDA_PATH}"
else
echo "Cudnn will not be linked against"
fi
mvn --version
cmake --version
protoc --version
nvcc --version
sudo apt-get autoremove
sudo apt-get clean
# download libnd4j from a url and set it up if LIBND4J_FILE_NAME is defined
bash ./bootstrap-libnd4j-from-url.sh linux cuda ${{ matrix.cuda_version }} "${{ matrix.helper }}"
if [ "$PERFORM_RELEASE" == 1 ]; then
echo "Performing release"
bash ${GITHUB_WORKSPACE}/release-specified-component.sh "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${COMMAND}"
else
echo "Running build and deploying to snapshots with command ${COMMAND}"
eval "${COMMAND}"
fi