azure-pipelines-data.yml
trigger:
branches:
include:
- master
- 'v*'
resources:
repositories:
- repository: LiberTEM-live
type: github
endpoint: LiberTEM
name: LiberTEM/LiberTEM-live
variables:
- name: PIP_CACHE_DIR
value: $(Pipeline.Workspace)/.pip
- group: Packaging
- group: Coverage
- name: npm_config_cache
value: $(Pipeline.Workspace)/.npm
- name: tox_dir
value: $(Build.Repository.LocalPath)/.tox
parameters:
# by default, the tox environment is kept around, so we need this parameter
# as a way to manually re-create it, in case dependencies change.
- name: recreate_tox
displayName: Recreate tox environments?
type: boolean
default: false
# This parameter allows to run slower tests, this is useful for running more
# comprehensive tests, for example before doing a release. This also enables more
# jobs to build on usually skipped Python versions.
- name: run_slow
displayName: Also run slow tests
type: boolean
default: false
stages:
- stage: test
jobs:
- job: check_example_notebooks
pool: DataAccess
variables:
TESTDATA_BASE_PATH: '/data/'
steps:
- bash: find /data/
displayName: 'list available data'
- task: UsePythonVersion@0
displayName: 'Use Python 3.9'
inputs:
versionSpec: '3.9'
# - task: Cache@2
# inputs:
# key: 'python | "$(Agent.OS)" | test_requirements.txt'
# restoreKeys: |
# python | "$(Agent.OS)"
# python
# path: $(PIP_CACHE_DIR)
# displayName: 'cache pip packages'
- bash: python3.9 -m venv venv
displayName: 'create venv'
- bash: ./venv/bin/pip install tox tox-uv
displayName: 'install tox'
- ${{ if eq(parameters.recreate_tox, true) }}:
- bash: ./venv/bin/tox -e notebooks -r -- examples/*.ipynb -v
displayName: 'Run nbval tests $(Agent.OS) (recreating)'
- ${{ if eq(parameters.recreate_tox, false) }}:
- bash: ./venv/bin/tox -e notebooks -- examples/*.ipynb -v
displayName: 'Run nbval tests $(Agent.OS)'
- bash: ./scripts/codecov.sh -f ./coverage.xml
displayName: 'Submit coverage to codecov.io'
${{ if and(not(eq(variables['Build.Reason'], 'PullRequest')), eq(variables['Build.SourceBranch'], 'refs/heads/master')) }}:
env:
CODECOV_TOKEN: $(CODECOV_TOKEN)
- task: PublishTestResults@2
condition: succeededOrFailed()
inputs:
testResultsFiles: 'junit.xml'
testRunTitle: 'Publish test results for Python $(python.version)'
- job: data_tests
pool: DataAccess
strategy:
matrix:
Python39:
python.version: '3.9'
TOXENV: 'py39-data'
Python310:
python.version: '3.10'
TOXENV: 'py310-data'
Python311:
python.version: '3.11'
TOXENV: 'py311-data'
Python312:
python.version: '3.12'
TOXENV: 'py312-data'
variables:
TESTDATA_BASE_PATH: '/data/'
steps:
- bash: find /data/
displayName: 'list available data'
- task: UsePythonVersion@0
displayName: 'Use Python $(python.version)'
inputs:
versionSpec: '$(python.version)'
- bash: python3.9 -m venv venv
displayName: 'create venv'
- bash: ./venv/bin/pip install tox tox-uv
displayName: 'install tox (w/ uv)'
- ${{ if eq(parameters.recreate_tox, true) }}:
- bash: ./venv/bin/tox -r
displayName: 'Run tox tests $(TOXENV) $(Agent.OS) (recreating)'
- ${{ if eq(parameters.recreate_tox, false) }}:
- bash: ./venv/bin/tox
displayName: 'Run tox tests $(TOXENV) $(Agent.OS)'
- ${{ if eq(parameters.run_slow, true) }}:
- bash: ./venv/bin/tox -- -m "slow and not dist" --cov-append
displayName: 'Run tox slow tests $(TOXENV) $(Agent.OS)'
- bash: ./scripts/codecov.sh -f ./coverage.xml
displayName: 'Submit coverage to codecov.io'
${{ if and(not(eq(variables['Build.Reason'], 'PullRequest')), eq(variables['Build.SourceBranch'], 'refs/heads/master')) }}:
env:
CODECOV_TOKEN: $(CODECOV_TOKEN)
- task: PublishTestResults@2
condition: succeededOrFailed()
inputs:
testResultsFiles: 'junit.xml'
testRunTitle: 'Publish test results for Python $(python.version)'
- job: numba_coverage
pool: DataAccess
variables:
TOXENV: 'numba_coverage'
steps:
- task: UsePythonVersion@0
displayName: 'Use Python 3.9'
inputs:
versionSpec: '3.9'
- bash: pip install -U tox tox-uv
displayName: 'install requirements'
- ${{ if eq(parameters.recreate_tox, true) }}:
- bash: tox -r
displayName: 'Run tox tests $(TOXENV) $(Agent.OS) (recreating)'
- ${{ if eq(parameters.recreate_tox, false) }}:
- bash: tox
displayName: 'Run tox tests $(TOXENV) $(Agent.OS)'
- bash: ./scripts/codecov.sh -f ./coverage.xml
displayName: 'Submit coverage to codecov.io'
${{ if and(not(eq(variables['Build.Reason'], 'PullRequest')), eq(variables['Build.SourceBranch'], 'refs/heads/master')) }}:
env:
CODECOV_TOKEN: $(CODECOV_TOKEN)
- task: PublishTestResults@2
condition: succeededOrFailed()
inputs:
testResultsFiles: 'junit.xml'
testRunTitle: 'Publish test results for numba coverage on Python 3.9'
- job: lint
pool: DataAccess
strategy:
matrix:
docs-check:
TOXENV: 'docs-check'
variables:
TOXENV: '$(TOXENV)'
steps:
- task: UsePythonVersion@0
displayName: 'Use Python 3.9'
inputs:
versionSpec: '3.9'
- bash: apt-get update && apt-get install -y pandoc graphviz
displayName: install required debian packages
- bash: pip install -U tox tox-uv
displayName: 'install requirements'
- ${{ if eq(parameters.recreate_tox, true) }}:
- bash: tox -r
displayName: 'Run tox tests $(TOXENV) $(Agent.OS) (recreating)'
- ${{ if eq(parameters.recreate_tox, false) }}:
- bash: tox
displayName: 'Run tox tests $(TOXENV) $(Agent.OS)'
- job: integration_live
pool: DataAccess
variables:
TESTDATA_BASE_PATH: '/data/'
steps:
- task: UsePythonVersion@0
displayName: 'Use Python 3.9'
inputs:
versionSpec: '3.9'
- checkout: self
- checkout: LiberTEM-live
- bash: python3 -m venv $(Agent.TempDirectory)/venv/
displayName: create virtualenv
- bash: $(Agent.TempDirectory)/venv/bin/pip install uv
displayName: install uv
# FIXME: install with uv, which bugs out with the path right now...
- bash: $(Agent.TempDirectory)/venv/bin/pip install $(Build.SourcesDirectory)/LiberTEM/
displayName: install LiberTEM
- bash: $(Agent.TempDirectory)/venv/bin/pip install "$(Build.SourcesDirectory)/LiberTEM-live/[dectris]"
displayName: install LiberTEM-live
- bash: . $(Agent.TempDirectory)/venv/bin/activate && $(Agent.TempDirectory)/venv/bin/uv pip install -r $(Build.SourcesDirectory)/LiberTEM-live/test_requirements.txt
displayName: install LiberTEM-live test requirements
- bash: cd $(Build.SourcesDirectory)/LiberTEM-live/ && $(Agent.TempDirectory)/venv/bin/pytest
displayName: run LiberTEM-live tests