.github/workflows/build-deploy-windows-cuda-11.6.yml
on:
workflow_dispatch:
inputs:
buildThreads:
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
required: true
default: 4
deployToReleaseStaging:
description: 'Whether to deploy to release staging or not.'
required: false
default: 0
releaseVersion:
description: 'Release version target'
required: false
default: 1.0.0-M3
snapshotVersion:
description: 'Snapshot version target'
required: false
default: 1.0.0-SNAPSHOT
releaseRepoId:
description: 'Release repository id'
required: false
default:
serverId:
description: 'Server id to publish to'
required: false
default: ossrh
mvnFlags:
description: 'Extra maven flags (must escape input yourself if used)'
required: false
default:
libnd4jUrl:
description: 'Whether to download libnd4j using https://github.com/KonduitAI/gh-actions-libnd4j-urls/ for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
required: false
default:
runsOn:
description: 'System to run on'
required: false
default: windows-2019
debug_enabled:
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
required: false
default: false
jobs:
windows-x86_64-cuda-11-6:
strategy:
fail-fast: false
matrix:
helper: [ cudnn,"" ]
extension: [ "" ]
include:
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
experimental: true
name: Extra maven flags
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
experimental: true
name: Debug enabled
- runs_on: ${{ github.event.inputs.runsOn }}
experimental: true
name: OS to run on
- libnd4j_file_download: ${{ github.event.inputs.libnd4jUrl }}
experimental: true
name: OS to run on
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
experimental: true
name: Whether to deploy to release staging or not
- release_version: ${{ github.event.inputs.releaseVersion }}
experimental: true
name: Release version
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
experimental: true
name: Snapshot version
- server_id: ${{ github.event.inputs.serverId }}
experimental: true
name: Server id
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
experimental: true
name: The release repository to run on
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
experimental: true
name: Extra maven flags to use as part of the build
- build_threads: ${{ github.event.inputs.buildThreads }}
experimental: true
name: The number of threads to build libnd4j with
runs-on: ${{ matrix.runs_on }}
steps:
- name: Cancel Previous Runs
uses: styfle/cancel-workflow-action@0.8.0
with:
access_token: ${{ github.token }}
- uses: actions/checkout@v2
- name: Set mvn build command based on matrix
shell: powershell
run: |
if ( "${{ matrix.libnd4j_file_download }}" -ne "" ) {
$modules=" :nd4j-cuda-11.6-preset,:nd4j-cuda-11.6"
} elseif ( "${{ matrix.helper }}" -ne "" ) {
$modules=":nd4j-cuda-11.6-preset,:nd4j-cuda-11.6,libnd4j"
} elseif ( "${{ matrix.helper }}" -eq "" ) {
$modules=":nd4j-cuda-11.6-preset,:nd4j-cuda-11.6,libnd4j,:nd4j-cuda-11.6-platform"
}
$command="mvn ${{ matrix.mvn_ext }} -Pcuda --also-make -pl $modules -Dlibnd4j.compute=`"5.0 5.2 6.1 7.5 8.6`" -Dlibnd4j.cpu.compile.skip=true -Dlibnd4j.buildthreads=${{ matrix.build_threads }} -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -Dlibnd4j.buildthreads=${{ github.event.inputs.buildThreads }} -Djavacpp.platform=windows-x86_64 -Dlibnd4j.platform=windows-x86_64 -Pcuda -Dlibnd4j.chip=cuda deploy -DskipTests"
if ( "${{ matrix.helper }}" -ne "" -And "${{ matrix.extension }}" -ne "" ) {
$mvn_ext=" -Dlibnd4j.classifier=windows-x86_64-cuda-11.6-${{ matrix.helper }}-${{matrix.extension}} -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.helper }}-${{ matrix.extension }} -Dlibnd4j.helper=${{ matrix.helper }} -Pcuda -Dlibnd4j.chip=cuda deploy -DskipTests"
$libnd4j_download_file_url="windows-cuda-11-${{ matrix.extension }}-${{ matrix.helper }}"
} elseif ( "${{ matrix.helper }}" -ne "" ) {
$mvn_ext=" -Dlibnd4j.classifier=windows-x86_64-cuda-11.6-${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.helper }} -Djavacpp.platform.extension=-${{ matrix.helper }} -Djavacpp.platform=windows-x86_64 -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.platform=windows-x86_64 -Pcuda -Dlibnd4j.chip=cuda deploy -DskipTests"
$libnd4j_download_file_url="windows-cuda-11-${{ matrix.helper }}"
} else {
$libnd4j_download_file_url="windows-cuda-11"
$mvn_ext=" -Dlibnd4j.classifier=windows-x86_64-cuda-11.6"
}
$command2= -join("$($command)","$($mvn_ext)");
$to_write = -join("COMMAND=","$($command2)");
if ( "${{ matrix.libnd4j_file_download }}" -ne "") {
echo "Adding libnd4j download"
$libnd4j_url_to_write = -join("LIBND4J_FILE_NAME=","$(${{ matrix.libnd4j_file_download }}/$libnd4j_download_file_url)");
echo $libnd4j_url_to_write | Out-File -FilePath "$env:GITHUB_ENV" -Encoding utf8 -Append
}
echo "Setting command for helper ${{ matrix.helper }} and extension ${{ matrix.extension }} to $($command2) and the libnd4j bootstrap file name to $($libnd4j_download_file_url)"
echo $command2 | Out-File -FilePath "$env:GITHUB_WORKSPACE/mvn-command.bat" -Encoding utf8 -Append
echo $to_write | Out-File -FilePath "$env:GITHUB_ENV" -Encoding utf8 -Append
echo "CUDNN_ROOT_DIR=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" | Out-File -FilePath "$env:GITHUB_ENV" -Encoding utf8 -Append
- name: Cache cuda install
uses: actions/cache@v2
id: cache-cuda-116
with:
path: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6
key: ${{ matrix.runs_on }}-cuda-11.6-${{ matrix.helper }}
restore-keys: ${{ matrix.runs_on }}-cuda-11.6-${{ matrix.helper }}
- name: Set up Java for publishing to GitHub Packages
uses: msys2/setup-msys2@v2
with:
msystem: MINGW64
update: true
install: base-devel git tar pkg-config unzip p7zip zip autoconf autoconf-archive automake patch mingw-w64-x86_64-make --noconfirm mingw-w64-x86_64-gnupg mingw-w64-x86_64-cmake mingw-w64-x86_64-nasm mingw-w64-x86_64-toolchain mingw-w64-x86_64-libtool mingw-w64-x86_64-gcc mingw-w64-x86_64-gcc-fortran mingw-w64-x86_64-libwinpthread-git mingw-w64-x86_64-SDL mingw-w64-x86_64-ragel
- uses: konduitai/cuda-install/.github/actions/install-cuda-windows@master
env:
cuda: 11.6.0
if: steps.cache-cuda-116.outputs.cache-hit != 'true'
- name: Setup tmate session
uses: mxschmitt/action-tmate@v3
if: ${{ github.event_name == 'workflow_dispatch' && matrix.debug_enabled }}
- name: Set up Java for publishing to OSSRH
uses: konduitai/setup-java@main
with:
java-version: 11
distribution: 'temurin'
server-id: ${{ github.event.inputs.serverId }}
server-username: MAVEN_USERNAME
server-password: MAVEN_PASSWORD
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
gpg-passphrase: MAVEN_GPG_PASSPHRASE
cache: 'maven'
- name: Setup windows path
shell: powershell
run: echo "C:\msys64\usr\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6\lib\x64;" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
- name: Import GPG Key
uses: crazy-max/ghaction-import-gpg@v1
env:
GPG_PRIVATE_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
- name: Run cuda build
shell: cmd
env:
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PUBLISH_TO: ossrh
MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }}
MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }}
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
PERFORM_RELEASE: ${{ matrix.deploy_to_release_staging }}
RELEASE_VERSION: ${{ matrix.release_version }}
SNAPSHOT_VERSION: ${{ matrix.snapshot_version }}
RELEASE_REPO_ID: ${{ matrix.release_repo_id }}
CUDA_PATH: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6
CUDNN_ROOT_DIR: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6
GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
GPG_SIGNING_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
MODULES: ${{ matrix.mvn_flags }}
LIBND4J_HOME_SUFFIX: cuda
HELPER: ${{ matrix.helper }}
EXTENSION: ${{ matrix.extension }}
MAVEN_OPTS: "-Xmx2g"
LIBND4J_FILE_NAME: ${{ matrix.libnd4j_file_download }}
run: |
echo "libnd4j build threads ${{ matrix.build_threads }}"
echo "deploy to release staging repo or not ${{ matrix.deploy_to_release_staging }}"
echo "release version ${{ matrix.release_version }}"
echo "snapshot version ${{ matrix.snapshot_version }}"
echo "debug enabled ${{ matrix.debug_enabled }}"
echo "libnd4j url ${{ matrix.libnd4j_file_download }}"
echo "maven flags ${{ matrix.mvn_flags }}"
echo "snapshot version ${{ matrix.snapshot_version }}"
echo "server id ${{ matrix.server_id }}"
echo "release repo id ${{ matrix.release_repo_id }}"
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
set MSYSTEM=MINGW64
echo "Running cuda build"
echo "Command is %COMMAND%"
bash ./change-cuda-versions.sh 11.6
Rem Note: we need this for the cudnn helpers, our cmake can't find it otherwise.
Rem See here: https://github.com/eclipse/deeplearning4j/blob/master/libnd4j/CMakeLists.txt#L298
set CUDNN_ROOT_DIR=%CUDA_PATH%
if "%PERFORM_RELEASE%"=="1" (
echo "Running release"
bash "%GITHUB_WORKSPACE%/bootstrap-libnd4j-from-url.sh" windows cuda 11.6 "${{ matrix.helper }}" "${{ matrix.extension }}"
bash "./release-specified-component.sh" "%RELEASE_VERSION%" "%SNAPSHOT_VERSION%" "%RELEASE_REPO_ID%"
) else (
if "%PERFORM_RELEASE%"==1 (
echo "Running release"
bash "%GITHUB_WORKSPACE%/bootstrap-libnd4j-from-url.sh"
bash "./release-specified-component.sh" "%RELEASE_VERSION%" "%SNAPSHOT_VERSION%" "%RELEASE_REPO_ID%"
) else (
echo "Running snapshots"
bash "%GITHUB_WORKSPACE%/bootstrap-libnd4j-from-url.sh"
call "%GITHUB_WORKSPACE%\mvn-command.bat"
)
)