.github/workflows/build.yml
---
# Ci code for building release artifacts.
name: Build
on:
push: # Master branch checks only validate the build and generate artifacts for testing.
branches:
- master
pull_request: null # PR checks only validate the build and generate artifacts for testing.
workflow_dispatch: # Dispatch runs build and validate, then push to the appropriate storage location.
inputs:
type:
description: Build Type
default: nightly
required: true
version:
description: Version Tag
default: nightly
required: true
concurrency: # This keeps multiple instances of the job from running concurrently for the same ref and event type.
group: build-${{ github.ref }}-${{ github.event_name }}
cancel-in-progress: true
jobs:
file-check: # Check what files changed if we’re being run in a PR or on a push.
name: Check Modified Files
runs-on: ubuntu-latest
outputs:
run: ${{ steps.check-run.outputs.run }}
skip-go: ${{ steps.check-go.outputs.skip-go }}
steps:
- name: Checkout
id: checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: recursive
- name: Check source files
id: check-source-files
uses: tj-actions/changed-files@v44
with:
since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
files: |
**/*.c
**/*.cc
**/*.h
**/*.hh
**/*.in
**/*.patch
src/aclk/aclk-schemas/
src/ml/dlib/
src/fluent-bit/
src/web/server/h2o/libh2o/
files_ignore: |
netdata.spec.in
**/*.md
- name: Check build files
id: check-build-files
uses: tj-actions/changed-files@v44
with:
since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
files: |
**/*.cmake
CMakeLists.txt
netdata-installer.sh
.github/data/distros.yml
.github/workflows/build.yml
.github/scripts/build-static.sh
.github/scripts/get-static-cache-key.sh
.github/scripts/gen-matrix-build.py
.github/scripts/run-updater-check.sh
packaging/cmake/
packaging/makeself/
packaging/installer/
packaging/*.sh
packaging/*.version
packaging/*.checksums
files_ignore: |
**/*.md
- name: List all changed files in pattern
continue-on-error: true
env:
CHANGED_SOURCE_FILES: ${{ steps.check-source-files.outputs.all_changed_files }}
CHANGED_BUILD_FILES: ${{ steps.check-build-files.outputs.all_changed_files }}
run: |
for file in ${CHANGED_SOURCE_FILES} ${CHANGED_BUILD_FILES} ; do
echo "$file was changed"
done
- name: Check Run
id: check-run
run: |
if [ "${{ steps.check-source-files.outputs.any_modified }}" == "true" ] || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
echo 'run=true' >> "${GITHUB_OUTPUT}"
else
echo 'run=false' >> "${GITHUB_OUTPUT}"
fi
- name: Check Go
id: check-go
env:
OTHER_CHANGED_FILES: ${{ steps.check-source-files.outputs.other_changed_files }}
run: |
if [ '${{ github.event_name }}' == 'pull_request' ]; then
if echo "${OTHER_CHANGED_FILES}" | grep -q '.*/(.*\.go|go\.mod|go\.sum)$' || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ]; then
echo 'skip-go=' >> "${GITHUB_OUTPUT}"
else
echo 'skip-go=--disable-go' >> "${GITHUB_OUTPUT}"
fi
else
echo 'skip-go=' >> "${GITHUB_OUTPUT}"
fi
build-dist: # Build the distribution tarball and store it as an artifact.
name: Build Distribution Tarball
runs-on: ubuntu-latest
needs:
- file-check
outputs:
distfile: ${{ steps.build.outputs.distfile }}
steps:
- name: Skip Check
id: skip
if: needs.file-check.outputs.run != 'true'
run: echo "SKIPPED"
- name: Checkout
id: checkout
if: needs.file-check.outputs.run == 'true'
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: recursive
- name: Fix tags
id: fix-tags
if: github.event_name != 'push' && needs.file-check.outputs.run == 'true'
run: |
git fetch --tags --force
- name: Mark Stable
id: channel
if: github.event_name == 'workflow_dispatch' && github.event.inputs.type != 'nightly' && needs.file-check.outputs.run == 'true'
run: |
sed -i 's/^RELEASE_CHANNEL="nightly"/RELEASE_CHANNEL="stable"/' netdata-installer.sh
- name: Build
id: build
if: needs.file-check.outputs.run == 'true'
run: |
mkdir -p artifacts/
tar --create --file "artifacts/netdata-$(git describe).tar.gz" \
--sort=name --posix --auto-compress --exclude=artifacts/ --exclude=.git \
--exclude=.gitignore --exclude=.gitattributes --exclude=.gitmodules \
--transform "s/^\\.\\//netdata-$(git describe)\\//" --verbose .
cd artifacts/
echo "distfile=$(find . -name 'netdata-*.tar.gz')" >> "${GITHUB_OUTPUT}"
- name: Store
id: store
if: needs.file-check.outputs.run == 'true'
uses: actions/upload-artifact@v4
with:
name: dist-tarball
path: artifacts/*.tar.gz
retention-days: 30
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Distribution tarball creation failed:'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Failed to create source tarball for distribution.
Checkout: ${{ steps.checkout.outcome }}
Fix Tags: ${{ steps.fix-tags.outcome }}
Mark stable: ${{ steps.channel.outcome }}
Build: ${{ steps.build.outcome }}
Store: ${{ steps.store.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& startsWith(github.ref, 'refs/heads/master')
&& github.event_name != 'pull_request'
&& github.repository == 'netdata/netdata'
&& needs.file-check.outputs.run == 'true'
}}
build-static: # Build the static binary archives, and store them as artifacts.
name: Build Static
runs-on: ubuntu-latest
needs:
- file-check
strategy:
matrix:
arch:
- x86_64
- armv6l
- armv7l
- aarch64
- ppc64le
steps:
- name: Skip Check
id: skip
if: needs.file-check.outputs.run != 'true'
run: echo "SKIPPED"
- name: Checkout
id: checkout
if: needs.file-check.outputs.run == 'true'
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: recursive
- name: Fix tags
id: fix-tags
if: github.event_name != 'push' && needs.file-check.outputs.run == 'true'
run: |
git fetch --tags --force
- name: Mark Stable
id: channel
if: github.event_name == 'workflow_dispatch' && github.event.inputs.type != 'nightly' && needs.file-check.outputs.run == 'true'
run: |
sed -i 's/^RELEASE_CHANNEL="nightly"/RELEASE_CHANNEL="stable"/' netdata-installer.sh packaging/makeself/install-or-update.sh
- name: Get Cache Key
if: (github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'run-ci/no-cache')) && needs.file-check.outputs.run == 'true'
id: cache-key
run: .github/scripts/get-static-cache-key.sh ${{ matrix.arch }} "${{ contains(github.event.pull_request.labels.*.name, 'run-ci/no-cache') }}"
- name: Cache
if: (github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'run-ci/no-cache')) && needs.file-check.outputs.run == 'true'
id: cache
uses: actions/cache@v4
with:
path: artifacts/cache
key: ${{ steps.cache-key.outputs.key }}
- name: Build
if: github.event_name != 'workflow_dispatch' && needs.file-check.outputs.run == 'true' # Don’t use retries on PRs.
run: |
export EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }}
.github/scripts/build-static.sh ${{ matrix.arch }}
- name: Build
if: github.event_name == 'workflow_dispatch' && needs.file-check.outputs.run == 'true'
id: build
uses: nick-fields/retry@v3
with:
timeout_minutes: 180
max_attempts: 3
command: |
export EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }}
.github/scripts/build-static.sh ${{ matrix.arch }}
- name: Store
id: store
if: needs.file-check.outputs.run == 'true'
uses: actions/upload-artifact@v4
with:
name: dist-static-${{ matrix.arch }}
path: artifacts/*.gz.run
retention-days: 30
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Static build failed:'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Failed to create static installer archive for ${{ matrix.arch }}.
Checkout: ${{ steps.checkout.outcome }}
Fix Tags: ${{ steps.fix-tags.outcome }}
Mark stable: ${{ steps.channel.outcome }}
Build: ${{ steps.build.outcome }}
Store: ${{ steps.store.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& startsWith(github.ref, 'refs/heads/master')
&& github.event_name != 'pull_request'
&& github.repository == 'netdata/netdata'
&& needs.file-check.outputs.run == 'true'
}}
matrix: # Generate the shared build matrix for our build tests.
name: Prepare Build Matrix
runs-on: ubuntu-latest
if: github.event_name != 'workflow_dispatch'
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- name: Checkout
id: checkout
uses: actions/checkout@v4
- name: Prepare tools
id: prepare
run: |
sudo apt-get update || true
sudo apt-get install -y python3-ruamel.yaml
- name: Read build matrix
id: set-matrix
run: |
matrix="$(.github/scripts/gen-matrix-build.py)"
echo "Generated matrix: ${matrix}"
echo "matrix=${matrix}" >> "${GITHUB_OUTPUT}"
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Build matrix preparation failed:'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Failed to prepare build matrix for build checks.
Checkout: ${{ steps.checkout.outcome }}
Prepare tools: ${{ steps.prepare.outcome }}
Read build matrix: ${{ steps.set-matrix.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& startsWith(github.ref, 'refs/heads/master')
&& github.event_name != 'pull_request'
&& github.repository == 'netdata/netdata'
}}
prepare-test-images: # Prepare the test environments for our build checks. This also checks dependency handling code for each tested environment.
name: Prepare Test Environments
runs-on: ubuntu-latest
if: github.event_name != 'workflow_dispatch'
needs:
- matrix
env:
RETRY_DELAY: 300
strategy:
# Unlike the actual build tests, this completes _very_ fast (average of about 3 minutes for each job), so we
# just run everything in parallel instead lof limiting job concurrency.
fail-fast: false
matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
steps:
- name: Checkout
id: checkout
uses: actions/checkout@v4
- name: Setup Buildx
id: buildx
uses: docker/setup-buildx-action@v3
- name: Build test environment
id: build1
uses: docker/build-push-action@v5
continue-on-error: true # We retry 3 times at 5 minute intervals if there is a failure here.
with:
push: false
load: false
file: .github/dockerfiles/Dockerfile.build_test
build-args: |
BASE=${{ matrix.distro }}
PRE=${{ matrix.env_prep }}
RMJSONC=${{ matrix.jsonc_removal }}
outputs: type=docker,dest=/tmp/image.tar
tags: test:${{ matrix.artifact_key }}
- name: Retry delay
if: ${{ steps.build1.outcome == 'failure' }}
run: sleep "${RETRY_DELAY}"
- name: Build test environment (attempt 2)
if: ${{ steps.build1.outcome == 'failure' }}
id: build2
uses: docker/build-push-action@v5
continue-on-error: true # We retry 3 times at 5 minute intervals if there is a failure here.
with:
push: false
load: false
file: .github/dockerfiles/Dockerfile.build_test
build-args: |
BASE=${{ matrix.distro }}
PRE=${{ matrix.env_prep }}
RMJSONC=${{ matrix.jsonc_removal }}
outputs: type=docker,dest=/tmp/image.tar
tags: test:${{ matrix.artifact_key }}
- name: Retry delay
if: ${{ steps.build1.outcome == 'failure' && steps.build2.outcome == 'failure' }}
run: sleep "${RETRY_DELAY}"
- name: Build test environment (attempt 3)
if: ${{ steps.build1.outcome == 'failure' && steps.build2.outcome == 'failure' }}
id: build3
uses: docker/build-push-action@v5
with:
push: false
load: false
file: .github/dockerfiles/Dockerfile.build_test
build-args: |
BASE=${{ matrix.distro }}
PRE=${{ matrix.env_prep }}
RMJSONC=${{ matrix.jsonc_removal }}
outputs: type=docker,dest=/tmp/image.tar
tags: test:${{ matrix.artifact_key }}
- name: Upload image artifact
id: upload
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.artifact_key }}-test-env
path: /tmp/image.tar
retention-days: 30
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Test environment preparation for ${{ matrix.distro }} failed:'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Test environment preparation for ${{ matrix.distro }} failed.
Checkout: ${{ steps.checkout.outcome }}
Set up Buildx: ${{ steps.buildx.outcome }}
Build test environment: ${{ steps.build1.outcome }}
Build test environment (attempt 2): ${{ steps.build2.outcome }}
Build test environment (attempt 3): ${{ steps.build3.outcome }}
Upload: ${{ steps.upload.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& startsWith(github.ref, 'refs/heads/master')
&& github.event_name != 'pull_request'
&& github.repository == 'netdata/netdata'
}}
source-build: # Test various source build arrangements.
name: Test Source Build
runs-on: ubuntu-latest
if: github.event_name != 'workflow_dispatch'
needs:
- matrix
- prepare-test-images
- file-check
strategy:
fail-fast: false
max-parallel: 8
matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
steps:
- name: Skip Check
id: skip
if: needs.file-check.outputs.run != 'true'
run: echo "SKIPPED"
- name: Checkout
id: checkout
if: needs.file-check.outputs.run == 'true'
uses: actions/checkout@v4
with:
submodules: recursive
- name: Fetch test environment
id: fetch
if: needs.file-check.outputs.run == 'true'
uses: Wandalen/wretry.action@v3
with:
action: actions/download-artifact@v4
with: |
name: ${{ matrix.artifact_key }}-test-env
path: .
attempt_limit: 3
attempt_delay: 2000
- name: Load test environment
id: load
if: needs.file-check.outputs.run == 'true'
run: docker load --input image.tar
- name: netdata-installer on ${{ matrix.distro }}, disable cloud
id: build-no-cloud
if: needs.file-check.outputs.run == 'true'
run: |
docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \
/bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --disable-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}'
- name: netdata-installer on ${{ matrix.distro }}, require cloud
id: build-cloud
if: needs.file-check.outputs.run == 'true'
run: |
docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \
/bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}'
- name: netdata-installer on ${{ matrix.distro }}, require cloud, no JSON-C
id: build-no-jsonc
if: matrix.jsonc_removal != '' && needs.file-check.outputs.run == 'true'
run: |
docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \
/bin/sh -c '/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}'
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Build tests for ${{ matrix.distro }} failed:'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Build tests for ${{ matrix.distro }} failed.
Checkout: ${{ steps.checkout.outcome }}
Fetch test environment: ${{ steps.fetch.outcome }}
Load test environment: ${{ steps.load.outcome }}
netdata-installer, disable cloud: ${{ steps.build-no-cloud.outcome }}
netdata-installer, require cloud: ${{ steps.build-cloud.outcome }}
netdata-installer, no JSON-C: ${{ steps.build-no-jsonc.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& startsWith(github.ref, 'refs/heads/master')
&& github.event_name != 'pull_request'
&& github.repository == 'netdata/netdata'
&& needs.file-check.outputs.run == 'true'
}}
updater-check: # Test the generated dist archive using the updater code.
name: Test Generated Distfile and Updater Code
runs-on: ubuntu-latest
if: github.event_name != 'workflow_dispatch'
needs:
- build-dist
- matrix
- prepare-test-images
- file-check
strategy:
fail-fast: false
max-parallel: 8
matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
services:
apache: # This gets used to serve the dist tarball for the updater script.
image: httpd:2.4
ports:
- 8080:80
volumes:
- ${{ github.workspace }}:/usr/local/apache2/htdocs/
steps:
- name: Skip Check
id: skip
if: needs.file-check.outputs.run != 'true'
run: echo "SKIPPED"
- name: Checkout
id: checkout
if: needs.file-check.outputs.run == 'true'
uses: actions/checkout@v4
- name: Fetch dist tarball artifacts
id: fetch-tarball
if: needs.file-check.outputs.run == 'true'
uses: Wandalen/wretry.action@v3
with:
action: actions/download-artifact@v4
with: |
name: dist-tarball
path: dist-tarball
attempt_limit: 3
attempt_delay: 2000
- name: Prepare artifact directory
id: prepare
if: needs.file-check.outputs.run == 'true'
run: |
mkdir -p artifacts/download/v9999.0.0 || exit 1
mkdir -p artifacts/latest || exit 1
echo "v9999.0.0" > artifacts/latest/latest-version.txt || exit 1
cp dist-tarball/* artifacts/download/v9999.0.0 || exit 1
cd artifacts/download/v9999.0.0 || exit 1
ln -s ${{ needs.build-dist.outputs.distfile }} netdata-latest.tar.gz || exit 1
ls -lFh
sha256sum -b ./* > "sha256sums.txt" || exit 1
cat sha256sums.txt
cd ../.. || exit 1
ls -lR
- name: Fetch test environment
id: fetch-test-environment
if: needs.file-check.outputs.run == 'true'
uses: Wandalen/wretry.action@v3
with:
action: actions/download-artifact@v4
with: |
name: ${{ matrix.artifact_key }}-test-env
path: .
attempt_limit: 3
attempt_delay: 2000
- name: Load test environment
id: load
if: needs.file-check.outputs.run == 'true'
run: docker load --input image.tar
- name: Install netdata and run the updater on ${{ matrix.distro }}
id: updater-check
if: needs.file-check.outputs.run == 'true'
run: |
docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 --network host -w /netdata \
-e EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }} \
test:${{ matrix.artifact_key }} /netdata/.github/scripts/run-updater-check.sh
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Updater checks for ${{ matrix.distro }} failed:'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Updater checks for ${{ matrix.distro }} failed.
Checkout: ${{ steps.checkout.outcome }}
Fetch dist tarball: ${{ steps.fetch-tarball.outcome }}
Prepare artifact directory: ${{ steps.prepare.outcome }}
Fetch test environment: ${{ steps.fetch-test-environment.outcome }}
Load test environment: ${{ steps.load.outcome }}
Updater check: ${{ steps.updater-check.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& startsWith(github.ref, 'refs/heads/master')
&& github.event_name != 'pull_request'
&& github.repository == 'netdata/netdata'
&& needs.file-check.outputs.run == 'true'
}}
prepare-upload: # Consolidate the artifacts for uploading or releasing.
name: Prepare Artifacts
runs-on: ubuntu-latest
needs:
- build-dist
- build-static
- file-check
steps:
- name: Skip Check
id: skip
if: needs.file-check.outputs.run != 'true'
run: echo "SKIPPED"
- name: Checkout
id: checkout
if: needs.file-check.outputs.run == 'true'
uses: actions/checkout@v4
- name: Prepare Environment
id: prepare
if: needs.file-check.outputs.run == 'true'
run: mkdir -p artifacts
- name: Retrieve Build Artifacts
id: fetch-dist
if: needs.file-check.outputs.run == 'true'
uses: Wandalen/wretry.action@v3
with:
action: actions/download-artifact@v4
with: |
pattern: dist-*
path: dist-artifacts
merge-multiple: true
attempt_limit: 3
attempt_delay: 2000
- name: Prepare Artifacts
id: consolidate
if: needs.file-check.outputs.run == 'true'
working-directory: ./artifacts/
run: |
mv ../dist-artifacts/* . || exit 1
ln -s ${{ needs.build-dist.outputs.distfile }} netdata-latest.tar.gz || exit 1
cp ../packaging/version ./latest-version.txt || exit 1
cp ../integrations/integrations.js ./integrations.js || exit 1
sha256sum -b ./* > sha256sums.txt || exit 1
cat sha256sums.txt
- name: Store Artifacts
id: store
if: needs.file-check.outputs.run == 'true'
uses: actions/upload-artifact@v4
with:
name: final-artifacts
path: artifacts/*
retention-days: 30
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Failed to prepare release artifacts for upload:'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Failed to prepare release artifacts for upload.
Checkout: ${{ steps.checkout.outcome }}
Prepare environment: ${{ steps.prepare.outcome }}
Fetch dist tarball: ${{ steps.fetch-dist.outcome }}
Consolidate artifacts: ${{ steps.consolidate.outcome }}
Store: ${{ steps.store.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& startsWith(github.ref, 'refs/heads/master')
&& github.event_name != 'pull_request'
&& github.repository == 'netdata/netdata'
&& needs.file-check.outputs.run == 'true'
}}
artifact-verification-dist: # Verify the regular installer works with the consolidated artifacts.
name: Test Consolidated Artifacts (Source)
runs-on: ubuntu-latest
needs:
- prepare-upload
- file-check
services:
apache: # This gets used to serve the dist tarball for the updater script.
image: httpd:2.4
ports:
- 8080:80
volumes:
- ${{ github.workspace }}:/usr/local/apache2/htdocs/
steps:
- name: Skip Check
id: skip
if: needs.file-check.outputs.run != 'true'
run: echo "SKIPPED"
- name: Checkout
id: checkout
if: needs.file-check.outputs.run == 'true'
uses: actions/checkout@v4
- name: Fetch artifacts
id: fetch
if: needs.file-check.outputs.run == 'true'
uses: Wandalen/wretry.action@v3
with:
action: actions/download-artifact@v4
with: |
name: final-artifacts
path: artifacts
attempt_limit: 3
attempt_delay: 2000
- name: Prepare artifacts directory
id: prepare
if: needs.file-check.outputs.run == 'true'
run: |
mkdir -p download/latest
mv artifacts/* download/latest
ls -al download/latest
- name: Verify that artifacts work with installer
id: verify
if: needs.file-check.outputs.run == 'true'
env:
NETDATA_TARBALL_BASEURL: http://localhost:8080/
run: packaging/installer/kickstart.sh --build-only --dont-start-it --disable-telemetry --dont-wait
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Artifact verification for source tarball failed.'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Artifact verification for source tarball failed.
Checkout: ${{ steps.checkout.outcome }}
Fetch artifacts: ${{ steps.fetch.outcome }}
Verify artifacts: ${{ steps.verify.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& startsWith(github.ref, 'refs/heads/master')
&& github.event_name != 'pull_request'
&& github.repository == 'netdata/netdata'
&& needs.file-check.outputs.run == 'true'
}}
artifact-verification-static: # Verify the static installer works with the consolidated artifacts.
name: Test Consolidated Artifacts (Static)
runs-on: ubuntu-latest
needs:
- prepare-upload
- file-check
services:
apache: # This gets used to serve the static archives.
image: httpd:2.4
ports:
- 8080:80
volumes:
- ${{ github.workspace }}:/usr/local/apache2/htdocs/
steps:
- name: Skip Check
id: skip
if: needs.file-check.outputs.run != 'true'
run: echo "SKIPPED"
- name: Checkout
id: checkout
if: needs.file-check.outputs.run == 'true'
uses: actions/checkout@v4
- name: Fetch artifacts
id: fetch-artifacts
if: needs.file-check.outputs.run == 'true'
uses: Wandalen/wretry.action@v3
with:
action: actions/download-artifact@v4
with: |
name: final-artifacts
path: artifacts
attempt_limit: 3
attempt_delay: 2000
- name: Prepare artifacts directory
id: prepare
if: needs.file-check.outputs.run == 'true'
run: |
mkdir -p download/latest
mv artifacts/* download/latest
ls -al download/latest
- name: Verify that artifacts work with installer
id: verify
if: needs.file-check.outputs.run == 'true'
env:
NETDATA_TARBALL_BASEURL: http://localhost:8080/
run: packaging/installer/kickstart.sh --static-only --dont-start-it --disable-telemetry
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Artifact verification for static build failed.'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Artifact verification for static build failed.
Checkout: ${{ steps.checkout.outcome }}
Fetch artifacts: ${{ steps.fetch-artifacts.outcome }}
Verify artifacts: ${{ steps.verify.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& startsWith(github.ref, 'refs/heads/master')
&& github.event_name != 'pull_request'
&& github.repository == 'netdata/netdata'
&& needs.file-check.outputs.run == 'true'
}}
upload-nightly: # Upload the nightly build artifacts to GCS.
name: Upload Nightly Artifacts
runs-on: ubuntu-latest
if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'nightly' && github.repository == 'netdata/netdata'
needs:
- artifact-verification-dist
- artifact-verification-static
steps:
- name: Retrieve Artifacts
id: fetch
uses: Wandalen/wretry.action@v3
with:
action: actions/download-artifact@v4
with: |
name: final-artifacts
path: final-artifacts
attempt_limit: 3
attempt_delay: 2000
- name: Authenticate to GCS
id: gcs-auth
uses: google-github-actions/auth@v2
with:
project_id: ${{ secrets.GCP_NIGHTLY_STORAGE_PROJECT }}
credentials_json: ${{ secrets.GCS_STORAGE_SERVICE_KEY_JSON }}
- name: Setup GCS
id: gcs-setup
uses: google-github-actions/setup-gcloud@v2.0.1
- name: Upload Artifacts
id: upload
uses: google-github-actions/upload-cloud-storage@v2.0.0
with:
destination: ${{ secrets.GCP_NIGHTLY_STORAGE_BUCKET }}
gzip: false
path: ./final-artifacts/latest-version.txt
parent: false
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Failed to upload nightly release artifacts:'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Failed to upload nightly release artifacts.
Fetch artifacts: ${{ steps.fetch.outcome }}
Authenticatie GCS: ${{ steps.gcs-auth.outcome }}
Setup GCS: ${{ steps.gcs-setup.outcome }}
Upload artifacts: ${{ steps.upload.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& startsWith(github.ref, 'refs/heads/master')
&& github.event_name != 'pull_request'
}}
create-nightly: # Create a nightly build release in netdata/netdata-nightlies
name: Create Nightly Release
runs-on: ubuntu-latest
if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'nightly' && github.repository == 'netdata/netdata'
needs:
- artifact-verification-dist
- artifact-verification-static
steps:
- name: Checkout Main Repo
id: checkout-main
uses: actions/checkout@v4
with:
path: main
- name: Checkout Nightly Repo
id: checkout-nightly
uses: actions/checkout@v4
with:
repository: netdata/netdata-nightlies
path: nightlies
token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
- name: Retrieve Artifacts
id: fetch
uses: Wandalen/wretry.action@v3
with:
action: actions/download-artifact@v4
with: |
name: final-artifacts
path: final-artifacts
attempt_limit: 3
attempt_delay: 2000
- name: Prepare version info
id: version
run: |
# shellcheck disable=SC2129
echo "version=$(cat main/packaging/version)" >> "${GITHUB_OUTPUT}"
echo "commit=$(cd nightlies && git rev-parse HEAD)" >> "${GITHUB_OUTPUT}"
echo "date=$(date +%F)" >> "${GITHUB_OUTPUT}"
- name: Create Release
id: create-release
uses: ncipollo/release-action@v1
with:
allowUpdates: false
artifactErrorsFailBuild: true
artifacts: 'final-artifacts/sha256sums.txt,final-artifacts/netdata-*.tar.gz,final-artifacts/netdata-*.gz.run,final-artifacts/integrations.js'
owner: netdata
repo: netdata-nightlies
body: Netdata nightly build for ${{ steps.version.outputs.date }}.
commit: ${{ steps.version.outputs.commit }}
makeLatest: true
tag: ${{ steps.version.outputs.version }}
token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
- name: Checkout netdata main Repo # Checkout back to netdata/netdata repo to the update latest packaged versions
id: checkout-netdata
uses: actions/checkout@v4
with:
token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
- name: Init python environment for publish release metadata
uses: actions/setup-python@v5
id: init-python
with:
python-version: "3.12"
- name: Setup python environment
id: setup-python
run: |
pip install -r .github/scripts/modules/requirements.txt
- name: Check if the version is latest and published
id: check-latest-version
run: |
python .github/scripts/check_latest_versions.py ${{ steps.version.outputs.version }}
- name: SSH setup
id: ssh-setup
if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-latest-version.outputs.versions_needs_update == 'true'
uses: shimataro/ssh-key-action@v2
with:
key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }}
name: id_ecdsa
known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }}
- name: Sync newer releases
id: sync-releases
if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-latest-version.outputs.versions_needs_update == 'true'
run: |
.github/scripts/upload-new-version-tags.sh
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Failed to draft release:'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Failed to create nightly release or attach artifacts.
Checkout netdata/netdata: ${{ steps.checkout-main.outcome }}
Checkout netdata/netdata-nightlies: ${{ steps.checkout-nightly.outcome }}
Fetch artifacts: ${{ steps.fetch.outcome }}
Prepare version info: ${{ steps.version.outcome }}
Create release: ${{ steps.create-release.outcome }}
Checkout back netdata/netdata: ${{ steps.checkout-netdata.outcome }}
Init python environment: ${{ steps.init-python.outcome }}
Setup python environment: ${{ steps.setup-python.outcome }}
Check the nearly published release against the advertised: ${{ steps.check-latest-version.outcome }}
Setup ssh: ${{ steps.ssh-setup.outcome }}
Sync with the releases: ${{ steps.sync-releases.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& github.event_name == 'workflow_dispatch'
}}
normalize-tag: # Fix the release tag if needed
name: Normalize Release Tag
runs-on: ubuntu-latest
if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'release'
outputs:
tag: ${{ steps.tag.outputs.tag }}
steps:
- name: Normalize Tag
id: tag
run: |
if echo ${{ github.event.inputs.version }} | grep -qE '^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$'; then
echo "tag=v${{ github.event.inputs.version }}" >> "${GITHUB_OUTPUT}"
else
echo "tag=${{ github.event.inputs.version }}" >> "${GITHUB_OUTPUT}"
fi
upload-release: # Create the draft release and upload the build artifacts.
name: Create Release Draft
runs-on: ubuntu-latest
if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'release' && github.repository == 'netdata/netdata'
needs:
- artifact-verification-dist
- artifact-verification-static
- normalize-tag
steps:
- name: Checkout
id: checkout
uses: actions/checkout@v4
- name: Retrieve Artifacts
id: fetch
uses: Wandalen/wretry.action@v3
with:
action: actions/download-artifact@v4
with: |
name: final-artifacts
path: final-artifacts
attempt_limit: 3
attempt_delay: 2000
- name: Create Release
id: create-release
uses: ncipollo/release-action@v1
with:
allowUpdates: false
artifactErrorsFailBuild: true
artifacts: 'final-artifacts/sha256sums.txt,final-artifacts/netdata-*.tar.gz,final-artifacts/netdata-*.gz.run,final-artifacts/integrations.js'
draft: true
tag: ${{ needs.normalize-tag.outputs.tag }}
token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Failed to draft release:'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Failed to create draft release or attach artifacts.
Checkout: ${{ steps.checkout.outcome }}
Fetch artifacts: ${{ steps.fetch.outcome }}
Create draft release: ${{ steps.create-release.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& github.event_name == 'workflow_dispatch'
}}
- name: Success Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'good'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Created agent draft release:'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: "${{ github.repository }}: ${{ steps.create-release.outputs.html_url }}"
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
success()
&& github.event_name == 'workflow_dispatch'
}}