packaging/dag/nd.py
from typing import List
import enum
import os
import pathlib
import uuid
import dagger
import jinja2
import imageutils
class Platform:
def __init__(self, platform: str):
self.platform = dagger.Platform(platform)
def escaped(self) -> str:
return str(self.platform).removeprefix("linux/").replace("/", "_")
def __eq__(self, other):
if isinstance(other, Platform):
return self.platform == other.platform
elif isinstance(other, dagger.Platform):
return self.platform == other
else:
return NotImplemented
def __ne__(self, other):
return not (self == other)
def __hash__(self):
return hash(self.platform)
def __str__(self) -> str:
return str(self.platform)
SUPPORTED_PLATFORMS = set(
[
Platform("linux/x86_64"),
Platform("linux/arm64"),
Platform("linux/i386"),
Platform("linux/arm/v7"),
Platform("linux/arm/v6"),
Platform("linux/ppc64le"),
Platform("linux/s390x"),
Platform("linux/riscv64"),
]
)
SUPPORTED_DISTRIBUTIONS = set(
[
"alpine_3_18",
"alpine_3_19",
"amazonlinux2",
"centos7",
"centos-stream8",
"centos-stream9",
"debian10",
"debian11",
"debian12",
"fedora37",
"fedora38",
"fedora39",
"opensuse15.4",
"opensuse15.5",
"opensusetumbleweed",
"oraclelinux8",
"oraclelinux9",
"rockylinux8",
"rockylinux9",
"ubuntu20.04",
"ubuntu22.04",
"ubuntu23.04",
"ubuntu23.10",
]
)
class Distribution:
def __init__(self, display_name):
self.display_name = display_name
if self.display_name == "alpine_3_18":
self.docker_tag = "alpine:3.18"
self.builder = imageutils.build_alpine_3_18
self.platforms = SUPPORTED_PLATFORMS
elif self.display_name == "alpine_3_19":
self.docker_tag = "alpine:3.19"
self.builder = imageutils.build_alpine_3_19
self.platforms = SUPPORTED_PLATFORMS
elif self.display_name == "amazonlinux2":
self.docker_tag = "amazonlinux:2"
self.builder = imageutils.build_amazon_linux_2
self.platforms = SUPPORTED_PLATFORMS
elif self.display_name == "centos7":
self.docker_tag = "centos:7"
self.builder = imageutils.build_centos_7
self.platforms = SUPPORTED_PLATFORMS
elif self.display_name == "centos-stream8":
self.docker_tag = "quay.io/centos/centos:stream8"
self.builder = imageutils.build_centos_stream_8
self.platforms = SUPPORTED_PLATFORMS
elif self.display_name == "centos-stream9":
self.docker_tag = "quay.io/centos/centos:stream9"
self.builder = imageutils.build_centos_stream_9
self.platforms = SUPPORTED_PLATFORMS
elif self.display_name == "debian10":
self.docker_tag = "debian:10"
self.builder = imageutils.build_debian_10
self.platforms = SUPPORTED_PLATFORMS
elif self.display_name == "debian11":
self.docker_tag = "debian:11"
self.builder = imageutils.build_debian_11
self.platforms = SUPPORTED_PLATFORMS
elif self.display_name == "debian12":
self.docker_tag = "debian:12"
self.builder = imageutils.build_debian_12
self.platforms = SUPPORTED_PLATFORMS
elif self.display_name == "fedora37":
self.docker_tag = "fedora:37"
self.builder = imageutils.build_fedora_37
self.platforms = SUPPORTED_PLATFORMS
elif self.display_name == "fedora38":
self.docker_tag = "fedora:38"
self.builder = imageutils.build_fedora_38
self.platforms = SUPPORTED_PLATFORMS
elif self.display_name == "fedora39":
self.docker_tag = "fedora:39"
self.platforms = SUPPORTED_PLATFORMS
self.builder = imageutils.build_fedora_39
elif self.display_name == "opensuse15.4":
self.docker_tag = "opensuse/leap:15.4"
self.builder = imageutils.build_opensuse_15_4
self.platforms = SUPPORTED_PLATFORMS
elif self.display_name == "opensuse15.5":
self.docker_tag = "opensuse/leap:15.5"
self.builder = imageutils.build_opensuse_15_5
self.platforms = SUPPORTED_PLATFORMS
elif self.display_name == "opensusetumbleweed":
self.docker_tag = "opensuse/tumbleweed:latest"
self.builder = imageutils.build_opensuse_tumbleweed
self.platforms = SUPPORTED_PLATFORMS
elif self.display_name == "oraclelinux8":
self.docker_tag = "oraclelinux:8"
self.builder = imageutils.build_oracle_linux_8
self.platforms = SUPPORTED_PLATFORMS
elif self.display_name == "oraclelinux9":
self.docker_tag = "oraclelinux:9"
self.builder = imageutils.build_oracle_linux_9
self.platforms = SUPPORTED_PLATFORMS
elif self.display_name == "rockylinux8":
self.docker_tag = "rockylinux:8"
self.builder = imageutils.build_rocky_linux_8
self.platforms = SUPPORTED_PLATFORMS
elif self.display_name == "rockylinux9":
self.docker_tag = "rockylinux:9"
self.builder = imageutils.build_rocky_linux_9
self.platforms = SUPPORTED_PLATFORMS
elif self.display_name == "ubuntu20.04":
self.docker_tag = "ubuntu:20.04"
self.builder = imageutils.build_ubuntu_20_04
self.platforms = SUPPORTED_PLATFORMS
elif self.display_name == "ubuntu22.04":
self.docker_tag = "ubuntu:22.04"
self.builder = imageutils.build_ubuntu_22_04
self.platforms = SUPPORTED_PLATFORMS
elif self.display_name == "ubuntu23.04":
self.docker_tag = "ubuntu:23.04"
self.builder = imageutils.build_ubuntu_23_04
self.platforms = SUPPORTED_PLATFORMS
elif self.display_name == "ubuntu23.10":
self.docker_tag = "ubuntu:23.10"
self.builder = imageutils.build_ubuntu_23_10
self.platforms = SUPPORTED_PLATFORMS
else:
raise ValueError(f"Unknown distribution: {self.display_name}")
def _cache_volume(
self, client: dagger.Client, platform: dagger.Platform, path: str
) -> dagger.CacheVolume:
tag = "_".join([self.display_name, Platform(platform).escaped()])
return client.cache_volume(f"{path}-{tag}")
def build(
self, client: dagger.Client, platform: dagger.Platform
) -> dagger.Container:
if platform not in self.platforms:
raise ValueError(
f"Building {self.display_name} is not supported on {platform}."
)
ctr = self.builder(client, platform)
ctr = imageutils.install_cargo(ctr)
return ctr
class FeatureFlags(enum.Flag):
DBEngine = enum.auto()
GoPlugin = enum.auto()
ExtendedBPF = enum.auto()
LogsManagement = enum.auto()
MachineLearning = enum.auto()
BundledProtobuf = enum.auto()
class NetdataInstaller:
def __init__(
self,
platform: Platform,
distro: Distribution,
repo_root: pathlib.Path,
prefix: pathlib.Path,
features: FeatureFlags,
):
self.platform = platform
self.distro = distro
self.repo_root = repo_root
self.prefix = prefix
self.features = features
def _mount_repo(
self, client: dagger.Client, ctr: dagger.Container, repo_root: pathlib.Path
) -> dagger.Container:
host_repo_root = pathlib.Path(__file__).parent.parent.parent.as_posix()
exclude_dirs = ["build", "fluent-bit/build", "packaging/dag"]
# The installer builds/stores intermediate artifacts under externaldeps/
# We add a volume to speed up rebuilds. The volume has to be unique
# per platform/distro in order to avoid mixing unrelated artifacts
# together.
externaldeps = self.distro._cache_volume(client, self.platform, "externaldeps")
ctr = (
ctr.with_directory(
self.repo_root.as_posix(), client.host().directory(host_repo_root)
)
.with_workdir(self.repo_root.as_posix())
.with_mounted_cache(
os.path.join(self.repo_root, "externaldeps"), externaldeps
)
)
return ctr
def install(self, client: dagger.Client, ctr: dagger.Container) -> dagger.Container:
args = ["--dont-wait", "--dont-start-it", "--disable-telemetry"]
if FeatureFlags.DBEngine not in self.features:
args.append("--disable-dbengine")
if FeatureFlags.GoPlugin not in self.features:
args.append("--disable-go")
if FeatureFlags.ExtendedBPF not in self.features:
args.append("--disable-ebpf")
if FeatureFlags.MachineLearning not in self.features:
args.append("--disable-ml")
if FeatureFlags.BundledProtobuf not in self.features:
args.append("--use-system-protobuf")
args.extend(["--install-prefix", self.prefix.parent.as_posix()])
ctr = self._mount_repo(client, ctr, self.repo_root.as_posix())
ctr = ctr.with_env_variable(
"NETDATA_CMAKE_OPTIONS", "-DCMAKE_BUILD_TYPE=Debug"
).with_exec(["./netdata-installer.sh"] + args)
return ctr
class Endpoint:
def __init__(self, hostname: str, port: int):
self.hostname = hostname
self.port = port
def __str__(self):
return ":".join([self.hostname, str(self.port)])
class ChildStreamConf:
def __init__(
self,
installer: NetdataInstaller,
destinations: List[Endpoint],
api_key: uuid.UUID,
):
self.installer = installer
self.substitutions = {
"enabled": "yes",
"destination": " ".join([str(dst) for dst in destinations]),
"api_key": api_key,
"timeout_seconds": 60,
"default_port": 19999,
"send_charts_matching": "*",
"buffer_size_bytes": 1024 * 1024,
"reconnect_delay_seconds": 5,
"initial_clock_resync_iterations": 60,
}
def render(self) -> str:
tmpl_path = pathlib.Path(__file__).parent / "files/child_stream.conf"
with open(tmpl_path) as fp:
tmpl = jinja2.Template(fp.read())
return tmpl.render(**self.substitutions)
class ParentStreamConf:
def __init__(self, installer: NetdataInstaller, api_key: uuid.UUID):
self.installer = installer
self.substitutions = {
"api_key": str(api_key),
"enabled": "yes",
"allow_from": "*",
"default_history": 3600,
"health_enabled_by_default": "auto",
"default_postpone_alarms_on_connect_seconds": 60,
"multiple_connections": "allow",
}
def render(self) -> str:
tmpl_path = pathlib.Path(__file__).parent / "files/parent_stream.conf"
with open(tmpl_path) as fp:
tmpl = jinja2.Template(fp.read())
return tmpl.render(**self.substitutions)
class StreamConf:
def __init__(self, child_conf: ChildStreamConf, parent_conf: ParentStreamConf):
self.child_conf = child_conf
self.parent_conf = parent_conf
def render(self) -> str:
child_section = self.child_conf.render() if self.child_conf else ""
parent_section = self.parent_conf.render() if self.parent_conf else ""
return "\n".join([child_section, parent_section])
class AgentContext:
def __init__(
self,
client: dagger.Client,
platform: dagger.Platform,
distro: Distribution,
installer: NetdataInstaller,
endpoint: Endpoint,
api_key: uuid.UUID,
allow_children: bool,
):
self.client = client
self.platform = platform
self.distro = distro
self.installer = installer
self.endpoint = endpoint
self.api_key = api_key
self.allow_children = allow_children
self.parent_contexts = []
self.built_distro = False
self.built_agent = False
def add_parent(self, parent_context: "AgentContext"):
self.parent_contexts.append(parent_context)
def build_container(self) -> dagger.Container:
ctr = self.distro.build(self.client, self.platform)
ctr = self.installer.install(self.client, ctr)
if len(self.parent_contexts) == 0 and not self.allow_children:
return ctr.with_exposed_port(self.endpoint.port)
destinations = [parent_ctx.endpoint for parent_ctx in self.parent_contexts]
child_stream_conf = ChildStreamConf(self.installer, destinations, self.api_key)
parent_stream_conf = None
if self.allow_children:
parent_stream_conf = ParentStreamConf(self.installer, self.api_key)
stream_conf = StreamConf(child_stream_conf, parent_stream_conf)
# write the stream conf to localhost and cp it in the container
host_stream_conf_path = pathlib.Path(
f"/tmp/{self.endpoint.hostname}_stream.conf"
)
with open(host_stream_conf_path, "w") as fp:
fp.write(stream_conf.render())
ctr_stream_conf_path = self.installer.prefix / "etc/netdata/stream.conf"
ctr = ctr.with_file(
ctr_stream_conf_path.as_posix(),
self.client.host().file(host_stream_conf_path.as_posix()),
)
ctr = ctr.with_exposed_port(self.endpoint.port)
return ctr