integrations/gen_docs_integrations.py
import json
import re
import shutil
from pathlib import Path
# Dictionary responsible for making the symbolic links at the end of the script's run.
symlink_dict = {}
def cleanup():
"""
clean directories that are either data collection or exporting integrations
"""
for element in Path("src/go/plugin/go.d/modules").glob('**/*/'):
if "integrations" in str(element):
shutil.rmtree(element)
for element in Path("src/collectors").glob('**/*/'):
# print(element)
if "integrations" in str(element):
shutil.rmtree(element)
for element in Path("src/exporting").glob('**/*/'):
if "integrations" in str(element):
shutil.rmtree(element)
for element in Path("integrations/cloud-notifications").glob('**/*/'):
if "integrations" in str(element) and not "metadata.yaml" in str(element):
shutil.rmtree(element)
for element in Path("integrations/logs").glob('**/*/'):
if "integrations" in str(element) and "metadata.yaml" not in str(element):
shutil.rmtree(element)
for element in Path("integrations/cloud-authentication").glob('**/*/'):
if "integrations" in str(element) and not "metadata.yaml" in str(element):
shutil.rmtree(element)
def generate_category_from_name(category_fragment, category_array):
"""
Takes a category ID in splitted form ("." as delimiter) and the array of the categories, and returns the proper category name that Learn expects.
"""
category_name = ""
i = 0
dummy_id = category_fragment[0]
while i < len(category_fragment):
for category in category_array:
if dummy_id == category['id']:
category_name = category_name + "/" + category["name"]
try:
# print("equals")
# print(fragment, category_fragment[i+1])
dummy_id = dummy_id + "." + category_fragment[i + 1]
# print(dummy_id)
except IndexError:
return category_name.split("/", 1)[1]
category_array = category['children']
break
i += 1
def clean_and_write(md, path):
"""
This function takes care of the special details element, and converts it to the equivalent that md expects.
Then it writes the buffer on the file provided.
"""
# clean first, replace
md = md.replace("{% details summary=\"", "<details><summary>")
md = md.replace("{% details open=true summary=\"", "<details open><summary>")
md = md.replace("\" %}", "</summary>\n")
md = md.replace("{% /details %}", "</details>\n")
path.write_text(md)
def add_custom_edit_url(markdown_string, meta_yaml_link, sidebar_label_string, mode='default'):
"""
Takes a markdown string and adds a "custom_edit_url" metadata to the metadata field
"""
output = ""
path_to_md_file = ""
if mode == 'default':
path_to_md_file = f'{meta_yaml_link.replace("/metadata.yaml", "")}/integrations/{clean_string(sidebar_label_string)}'
elif mode == 'cloud-notification':
path_to_md_file = meta_yaml_link.replace("metadata.yaml", f'integrations/{clean_string(sidebar_label_string)}')
elif mode == 'agent-notification':
path_to_md_file = meta_yaml_link.replace("metadata.yaml", "README")
elif mode == 'cloud-authentication':
path_to_md_file = meta_yaml_link.replace("metadata.yaml", f'integrations/{clean_string(sidebar_label_string)}')
elif mode == 'logs':
path_to_md_file = meta_yaml_link.replace("metadata.yaml", f'integrations/{clean_string(sidebar_label_string)}')
output = markdown_string.replace(
"<!--startmeta",
f'<!--startmeta\ncustom_edit_url: \"{path_to_md_file}.md\"')
return output
def clean_string(string):
"""
simple function to get rid of caps, spaces, slashes and parentheses from a given string
The string represents an integration name, as it would be displayed in the final text
"""
return string.lower().replace(" ", "_").replace("/", "-").replace("(", "").replace(")", "").replace(":", "")
def read_integrations_js(path_to_file):
"""
Open integrations/integrations.js and extract the dictionaries
"""
try:
data = Path(path_to_file).read_text()
categories_str = data.split("export const categories = ")[1].split("export const integrations = ")[0]
integrations_str = data.split("export const categories = ")[1].split("export const integrations = ")[1]
return json.loads(categories_str), json.loads(integrations_str)
except FileNotFoundError as e:
print("Exception", e)
def create_overview(integration, filename, overview_key_name="overview"):
# empty overview_key_name to have only image on overview
if not overview_key_name:
return f"""# {integration['meta']['name']}
<img src="https://netdata.cloud/img/{filename}" width="150"/>
"""
split = re.split(r'(#.*\n)', integration[overview_key_name], 1)
first_overview_part = split[1]
rest_overview_part = split[2]
if not filename:
return f"""{first_overview_part}{rest_overview_part}
"""
return f"""{first_overview_part}
<img src="https://netdata.cloud/img/{filename}" width="150"/>
{rest_overview_part}
"""
def build_readme_from_integration(integration, mode=''):
# COLLECTORS
if mode == 'collector':
try:
# initiate the variables for the collector
meta_yaml = integration['edit_link'].replace("blob", "edit")
sidebar_label = integration['meta']['monitored_instance']['name']
learn_rel_path = generate_category_from_name(
integration['meta']['monitored_instance']['categories'][0].split("."), categories).replace(
"Data Collection", "Collecting Metrics")
most_popular = integration['meta']['most_popular']
# build the markdown string
md = \
f"""<!--startmeta
meta_yaml: "{meta_yaml}"
sidebar_label: "{sidebar_label}"
learn_status: "Published"
learn_rel_path: "{learn_rel_path}"
most_popular: {most_popular}
message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE COLLECTOR'S metadata.yaml FILE"
endmeta-->
{create_overview(integration, integration['meta']['monitored_instance']['icon_filename'])}"""
if integration['metrics']:
md += f"""
{integration['metrics']}
"""
if integration['alerts']:
md += f"""
{integration['alerts']}
"""
if integration['setup']:
md += f"""
{integration['setup']}
"""
if integration['troubleshooting']:
md += f"""
{integration['troubleshooting']}
"""
except Exception as e:
print("Exception in collector md construction", e, integration['id'])
# EXPORTERS
elif mode == 'exporter':
try:
# initiate the variables for the exporter
meta_yaml = integration['edit_link'].replace("blob", "edit")
sidebar_label = integration['meta']['name']
learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories)
# build the markdown string
md = \
f"""<!--startmeta
meta_yaml: "{meta_yaml}"
sidebar_label: "{sidebar_label}"
learn_status: "Published"
learn_rel_path: "Exporting Metrics"
message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE EXPORTER'S metadata.yaml FILE"
endmeta-->
{create_overview(integration, integration['meta']['icon_filename'])}"""
if integration['setup']:
md += f"""
{integration['setup']}
"""
if integration['troubleshooting']:
md += f"""
{integration['troubleshooting']}
"""
except Exception as e:
print("Exception in exporter md construction", e, integration['id'])
# NOTIFICATIONS
elif mode == 'agent-notification':
try:
# initiate the variables for the notification method
meta_yaml = integration['edit_link'].replace("blob", "edit")
sidebar_label = integration['meta']['name']
learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories)
# build the markdown string
md = \
f"""<!--startmeta
meta_yaml: "{meta_yaml}"
sidebar_label: "{sidebar_label}"
learn_status: "Published"
learn_rel_path: "{learn_rel_path.replace("notifications", "Alerts & Notifications/Notifications")}"
message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE NOTIFICATION'S metadata.yaml FILE"
endmeta-->
{create_overview(integration, integration['meta']['icon_filename'], "overview")}"""
if integration['setup']:
md += f"""
{integration['setup']}
"""
if integration['troubleshooting']:
md += f"""
{integration['troubleshooting']}
"""
except Exception as e:
print("Exception in notification md construction", e, integration['id'])
elif mode == 'cloud-notification':
try:
# initiate the variables for the notification method
meta_yaml = integration['edit_link'].replace("blob", "edit")
sidebar_label = integration['meta']['name']
learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories)
# build the markdown string
md = \
f"""<!--startmeta
meta_yaml: "{meta_yaml}"
sidebar_label: "{sidebar_label}"
learn_status: "Published"
learn_rel_path: "{learn_rel_path.replace("notifications", "Alerts & Notifications/Notifications")}"
message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE NOTIFICATION'S metadata.yaml FILE"
endmeta-->
{create_overview(integration, integration['meta']['icon_filename'], "")}"""
if integration['setup']:
md += f"""
{integration['setup']}
"""
if integration['troubleshooting']:
md += f"""
{integration['troubleshooting']}
"""
except Exception as e:
print("Exception in notification md construction", e, integration['id'])
elif mode == 'logs':
try:
# initiate the variables for the logs integration
meta_yaml = integration['edit_link'].replace("blob", "edit")
sidebar_label = integration['meta']['name']
learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories)
# build the markdown string
md = \
f"""<!--startmeta
meta_yaml: "{meta_yaml}"
sidebar_label: "{sidebar_label}"
learn_status: "Published"
learn_rel_path: "{learn_rel_path.replace("logs", "Logs")}"
message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE LOGS' metadata.yaml FILE"
endmeta-->
{create_overview(integration, integration['meta']['icon_filename'])}"""
if integration['setup']:
md += f"""
{integration['setup']}
"""
except Exception as e:
print("Exception in logs md construction", e, integration['id'])
# AUTHENTICATIONS
elif mode == 'authentication':
if True:
# initiate the variables for the authentication method
meta_yaml = integration['edit_link'].replace("blob", "edit")
sidebar_label = integration['meta']['name']
learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories)
# build the markdown string
md = \
f"""<!--startmeta
meta_yaml: "{meta_yaml}"
sidebar_label: "{sidebar_label}"
learn_status: "Published"
learn_rel_path: "{learn_rel_path.replace("authentication", "Netdata Cloud/Authentication & Authorization/Cloud Authentication & Authorization Integrations")}"
message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE AUTHENTICATION'S metadata.yaml FILE"
endmeta-->
{create_overview(integration, integration['meta']['icon_filename'])}"""
if integration['setup']:
md += f"""
{integration['setup']}
"""
if integration['troubleshooting']:
md += f"""
{integration['troubleshooting']}
"""
# except Exception as e:
# print("Exception in authentication md construction", e, integration['id'])
if "community" in integration['meta'].keys():
community = "<img src=\"https://img.shields.io/badge/maintained%20by-Community-blue\" />"
else:
community = "<img src=\"https://img.shields.io/badge/maintained%20by-Netdata-%2300ab44\" />"
return meta_yaml, sidebar_label, learn_rel_path, md, community
def build_path(meta_yaml_link):
"""
function that takes a metadata yaml file link, and makes it into a path that gets used to write to a file.
"""
return meta_yaml_link.replace("https://github.com/netdata/", "") \
.split("/", 1)[1] \
.replace("edit/master/", "") \
.replace("/metadata.yaml", "")
def write_to_file(path, md, meta_yaml, sidebar_label, community, mode='default'):
"""
takes the arguments needed to write the integration markdown to the proper file.
"""
upper, lower = md.split("##", 1)
md = upper + community + f"\n\n##{lower}"
if mode == 'default':
# Only if the path exists, this caters for running the same script on both the go and netdata repos.
if Path(path).exists():
if not Path(f'{path}/integrations').exists():
Path(f'{path}/integrations').mkdir()
try:
md = add_custom_edit_url(md, meta_yaml, sidebar_label)
clean_and_write(
md,
Path(f'{path}/integrations/{clean_string(sidebar_label)}.md')
)
except FileNotFoundError as e:
print("Exception in writing to file", e)
# If we only created one file inside the directory, add the entry to the symlink_dict, so we can make the symbolic link
if len(list(Path(f'{path}/integrations').iterdir())) == 1:
symlink_dict.update(
{path: f'integrations/{clean_string(sidebar_label)}.md'})
else:
try:
symlink_dict.pop(path)
except KeyError:
# We don't need to print something here.
pass
elif mode == 'cloud-notification':
# for cloud notifications we generate them near their metadata.yaml
name = clean_string(integration['meta']['name'])
if not Path(f'{path}/integrations').exists():
Path(f'{path}/integrations').mkdir()
# proper_edit_name = meta_yaml.replace(
# "metadata.yaml", f'integrations/{clean_string(sidebar_label)}.md\"')
md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='cloud-notification')
finalpath = f'{path}/integrations/{name}.md'
try:
clean_and_write(
md,
Path(finalpath)
)
except FileNotFoundError as e:
print("Exception in writing to file", e)
elif mode == 'agent-notification':
# add custom_edit_url as the md file, so we can have uniqueness in the ingest script
# afterwards the ingest will replace this metadata with meta_yaml
md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='agent-notification')
finalpath = f'{path}/README.md'
try:
clean_and_write(
md,
Path(finalpath)
)
except FileNotFoundError as e:
print("Exception in writing to file", e)
elif mode == 'logs':
# for logs we generate them near their metadata.yaml
name = clean_string(integration['meta']['name'])
if not Path(f'{path}/integrations').exists():
Path(f'{path}/integrations').mkdir()
# proper_edit_name = meta_yaml.replace(
# "metadata.yaml", f'integrations/{clean_string(sidebar_label)}.md\"')
md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='logs')
finalpath = f'{path}/integrations/{name}.md'
try:
clean_and_write(
md,
Path(finalpath)
)
except FileNotFoundError as e:
print("Exception in writing to file", e)
elif mode == 'authentication':
name = clean_string(integration['meta']['name'])
if not Path(f'{path}/integrations').exists():
Path(f'{path}/integrations').mkdir()
# proper_edit_name = meta_yaml.replace(
# "metadata.yaml", f'integrations/{clean_string(sidebar_label)}.md\"')
md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='cloud-authentication')
finalpath = f'{path}/integrations/{name}.md'
try:
clean_and_write(
md,
Path(finalpath)
)
except FileNotFoundError as e:
print("Exception in writing to file", e)
def make_symlinks(symlink_dict):
"""
takes a dictionary with directories that have a 1:1 relationship between their README and the integration (only one) inside the "integrations" folder.
"""
for element in symlink_dict:
if not Path(f'{element}/README.md').exists():
Path(f'{element}/README.md').touch()
try:
# Remove the README to prevent it being a normal file
Path(f'{element}/README.md').unlink()
except FileNotFoundError:
continue
# and then make a symlink to the actual markdown
Path(f'{element}/README.md').symlink_to(symlink_dict[element])
filepath = Path(f'{element}/{symlink_dict[element]}')
md = filepath.read_text()
# This preserves the custom_edit_url for most files as it was,
# so the existing links don't break, this is vital for link replacement afterwards
filepath.write_text(md.replace(
f'{element}/{symlink_dict[element]}', f'{element}/README.md'))
cleanup()
categories, integrations = read_integrations_js('integrations/integrations.js')
# Iterate through every integration
for integration in integrations:
if integration['integration_type'] == "collector":
meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
integration, mode='collector')
path = build_path(meta_yaml)
write_to_file(path, md, meta_yaml, sidebar_label, community)
else:
# kind of specific if clause, so we can avoid running excessive code in the go repo
if integration['integration_type'] == "exporter":
meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
integration, mode='exporter')
path = build_path(meta_yaml)
write_to_file(path, md, meta_yaml, sidebar_label, community)
elif integration['integration_type'] == "agent_notification":
meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
integration, mode='agent-notification')
path = build_path(meta_yaml)
write_to_file(path, md, meta_yaml, sidebar_label, community, mode='agent-notification')
elif integration['integration_type'] == "cloud_notification":
meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
integration, mode='cloud-notification')
path = build_path(meta_yaml)
write_to_file(path, md, meta_yaml, sidebar_label, community, mode='cloud-notification')
elif integration['integration_type'] == "logs":
meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
integration, mode='logs')
path = build_path(meta_yaml)
write_to_file(path, md, meta_yaml, sidebar_label, community, mode='logs')
elif integration['integration_type'] == "authentication":
meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
integration, mode='authentication')
path = build_path(meta_yaml)
write_to_file(path, md, meta_yaml, sidebar_label, community, mode='authentication')
make_symlinks(symlink_dict)