scripts/make.py
# Support code for building docs (build_usage, build_man)
import glob
import os
import io
import re
import sys
import textwrap
from collections import OrderedDict
from datetime import datetime, timezone
import time
def format_metavar(option):
if option.nargs in ("*", "..."):
return "[%s...]" % option.metavar
elif option.nargs == "?":
return "[%s]" % option.metavar
elif option.nargs is None:
return option.metavar
else:
raise ValueError(f"Can't format metavar {option.metavar}, unknown nargs {option.nargs}!")
class BuildUsage:
"""generate usage docs for each command"""
def run(self):
print("generating usage docs")
import borg
borg.doc_mode = "build_man"
os.makedirs("docs/usage", exist_ok=True)
# allows us to build docs without the C modules fully loaded during help generation
from borg.archiver import Archiver
parser = Archiver(prog="borg").build_parser()
# borgfs has a separate man page to satisfy debian's "every program from a package
# must have a man page" requirement, but it doesn't need a separate HTML docs page
# borgfs_parser = Archiver(prog='borgfs').build_parser()
self.generate_level("", parser, Archiver)
return 0
def generate_level(self, prefix, parser, Archiver, extra_choices=None):
is_subcommand = False
choices = {}
for action in parser._actions:
if action.choices is not None and "SubParsersAction" in str(action.__class__):
is_subcommand = True
for cmd, parser in action.choices.items():
choices[prefix + cmd] = parser
if extra_choices is not None:
choices.update(extra_choices)
if prefix and not choices:
return
print("found commands: %s" % list(choices.keys()))
for command, parser in sorted(choices.items()):
if command.startswith("debug"):
print("skipping", command)
continue
print("generating help for %s" % command)
if self.generate_level(command + " ", parser, Archiver):
continue
with open("docs/usage/%s.rst.inc" % command.replace(" ", "_"), "w") as doc:
doc.write(".. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!\n\n")
if command == "help":
for topic in Archiver.helptext:
params = {"topic": topic, "underline": "~" * len("borg help " + topic)}
doc.write(".. _borg_{topic}:\n\n".format(**params))
doc.write("borg help {topic}\n{underline}\n\n".format(**params))
doc.write(Archiver.helptext[topic])
else:
params = {
"command": command,
"command_": command.replace(" ", "_"),
"underline": "-" * len("borg " + command),
}
doc.write(".. _borg_{command_}:\n\n".format(**params))
doc.write(
"borg {command}\n{underline}\n.. code-block:: none\n\n borg [common options] {command}".format(
**params
)
)
self.write_usage(parser, doc)
epilog = parser.epilog
parser.epilog = None
self.write_options(parser, doc)
doc.write("\n\nDescription\n~~~~~~~~~~~\n")
doc.write(epilog)
if "create" in choices:
common_options = [group for group in choices["create"]._action_groups if group.title == "Common options"][0]
with open("docs/usage/common-options.rst.inc", "w") as doc:
self.write_options_group(common_options, doc, False, base_indent=0)
return is_subcommand
def write_usage(self, parser, fp):
if any(len(o.option_strings) for o in parser._actions):
fp.write(" [options]")
for option in parser._actions:
if option.option_strings:
continue
fp.write(" " + format_metavar(option))
fp.write("\n\n")
def write_options(self, parser, fp):
def is_positional_group(group):
return any(not o.option_strings for o in group._group_actions)
# HTML output:
# A table using some column-spans
rows = []
for group in parser._action_groups:
if group.title == "Common options":
# (no of columns used, columns, ...)
rows.append((1, ".. class:: borg-common-opt-ref\n\n:ref:`common_options`"))
else:
if not group._group_actions:
continue
group_header = "**%s**" % group.title
if group.description:
group_header += " — " + group.description
rows.append((1, group_header))
if is_positional_group(group):
for option in group._group_actions:
rows.append((3, "", "``%s``" % option.metavar, option.help or ""))
else:
for option in group._group_actions:
if option.metavar:
option_fmt = "``%s " + option.metavar + "``"
else:
option_fmt = "``%s``"
option_str = ", ".join(option_fmt % s for s in option.option_strings)
option_desc = textwrap.dedent((option.help or "") % option.__dict__)
rows.append((3, "", option_str, option_desc))
fp.write(".. only:: html\n\n")
table = io.StringIO()
table.write(".. class:: borg-options-table\n\n")
self.rows_to_table(rows, table.write)
fp.write(textwrap.indent(table.getvalue(), " " * 4))
# LaTeX output:
# Regular rST option lists (irregular column widths)
latex_options = io.StringIO()
for group in parser._action_groups:
if group.title == "Common options":
latex_options.write("\n\n:ref:`common_options`\n")
latex_options.write(" |")
else:
self.write_options_group(group, latex_options)
fp.write("\n.. only:: latex\n\n")
fp.write(textwrap.indent(latex_options.getvalue(), " " * 4))
def rows_to_table(self, rows, write):
def write_row_separator():
write("+")
for column_width in column_widths:
write("-" * (column_width + 1))
write("+")
write("\n")
# Find column count and width
column_count = max(columns for columns, *_ in rows)
column_widths = [0] * column_count
for columns, *cells in rows:
for i in range(columns):
# "+ 1" because we want a space between the cell contents and the delimiting "|" in the output
column_widths[i] = max(column_widths[i], len(cells[i]) + 1)
for columns, *original_cells in rows:
write_row_separator()
# If a cell contains newlines, then the row must be split up in individual rows
# where each cell contains no newline.
rowspanning_cells = []
original_cells = list(original_cells)
while any("\n" in cell for cell in original_cells):
cell_bloc = []
for i, cell in enumerate(original_cells):
pre, _, original_cells[i] = cell.partition("\n")
cell_bloc.append(pre)
rowspanning_cells.append(cell_bloc)
rowspanning_cells.append(original_cells)
for cells in rowspanning_cells:
for i, column_width in enumerate(column_widths):
if i < columns:
write("| ")
write(cells[i].ljust(column_width))
else:
write(" ")
write("".ljust(column_width))
write("|\n")
write_row_separator()
# This bit of JavaScript kills the <colgroup> that is invariably inserted by docutils,
# but does absolutely no good here. It sets bogus column widths which cannot be overridden
# with CSS alone.
# Since this is HTML-only output, it would be possible to just generate a <table> directly,
# but then we'd lose rST formatting.
write(
textwrap.dedent(
"""
.. raw:: html
<script type='text/javascript'>
$(document).ready(function () {
$('.borg-options-table colgroup').remove();
})
</script>
"""
)
)
def write_options_group(self, group, fp, with_title=True, base_indent=4):
def is_positional_group(group):
return any(not o.option_strings for o in group._group_actions)
indent = " " * base_indent
if is_positional_group(group):
for option in group._group_actions:
fp.write(option.metavar + "\n")
fp.write(textwrap.indent(option.help or "", " " * base_indent) + "\n")
return
if not group._group_actions:
return
if with_title:
fp.write("\n\n")
fp.write(group.title + "\n")
opts = OrderedDict()
for option in group._group_actions:
if option.metavar:
option_fmt = "%s " + option.metavar
else:
option_fmt = "%s"
option_str = ", ".join(option_fmt % s for s in option.option_strings)
option_desc = textwrap.dedent((option.help or "") % option.__dict__)
opts[option_str] = textwrap.indent(option_desc, " " * 4)
padding = len(max(opts)) + 1
for option, desc in opts.items():
fp.write(indent + option.ljust(padding) + desc + "\n")
class BuildMan:
"""build man pages"""
see_also = {
"create": ("delete", "prune", "check", "patterns", "placeholders", "compression", "rcreate"),
"recreate": ("patterns", "placeholders", "compression"),
"list": ("info", "diff", "prune", "patterns", "rlist"),
"info": ("list", "diff", "rinfo"),
"rcreate": ("rdelete", "rlist", "check", "benchmark-cpu", "key-import", "key-export", "key-change-passphrase"),
"key-import": ("key-export",),
"key-export": ("key-import",),
"mount": ("umount", "extract"), # Would be cooler if these two were on the same page
"umount": ("mount",),
"extract": ("mount",),
"delete": ("compact", "rdelete"),
"prune": ("compact",),
}
rst_prelude = textwrap.dedent(
"""
.. role:: ref(title)
.. |project_name| replace:: Borg
"""
)
usage_group = {
"break-lock": "lock",
"with-lock": "lock",
"key_change-passphrase": "key",
"key_change-location": "key",
"key_export": "key",
"key_import": "key",
"export-tar": "tar",
"import-tar": "tar",
"benchmark_crud": "benchmark",
"benchmark_cpu": "benchmark",
"umount": "mount",
}
def run(self):
print("building man pages (in docs/man)", file=sys.stderr)
import borg
borg.doc_mode = "build_man"
os.makedirs("docs/man", exist_ok=True)
# allows us to build docs without the C modules fully loaded during help generation
from borg.archiver import Archiver
parser = Archiver(prog="borg").build_parser()
borgfs_parser = Archiver(prog="borgfs").build_parser()
self.generate_level("", parser, Archiver, {"borgfs": borgfs_parser})
self.build_topic_pages(Archiver)
self.build_intro_page()
return 0
def generate_level(self, prefix, parser, Archiver, extra_choices=None):
is_subcommand = False
choices = {}
for action in parser._actions:
if action.choices is not None and "SubParsersAction" in str(action.__class__):
is_subcommand = True
for cmd, parser in action.choices.items():
choices[prefix + cmd] = parser
if extra_choices is not None:
choices.update(extra_choices)
if prefix and not choices:
return
for command, parser in sorted(choices.items()):
if command.startswith("debug") or command == "help":
continue
if command == "borgfs":
man_title = command
else:
man_title = "borg-" + command.replace(" ", "-")
print("building man page", man_title + "(1)", file=sys.stderr)
is_intermediary = self.generate_level(command + " ", parser, Archiver)
doc, write = self.new_doc()
self.write_man_header(write, man_title, parser.description)
self.write_heading(write, "SYNOPSIS")
if is_intermediary:
subparsers = [action for action in parser._actions if "SubParsersAction" in str(action.__class__)][0]
for subcommand in subparsers.choices:
write("| borg", "[common options]", command, subcommand, "...")
self.see_also.setdefault(command, []).append(f"{command}-{subcommand}")
else:
if command == "borgfs":
write(command, end="")
else:
write("borg", "[common options]", command, end="")
self.write_usage(write, parser)
write("\n")
description, _, notes = parser.epilog.partition("\n.. man NOTES")
if description:
self.write_heading(write, "DESCRIPTION")
write(description)
if not is_intermediary:
self.write_heading(write, "OPTIONS")
write("See `borg-common(1)` for common options of Borg commands.")
write()
self.write_options(write, parser)
self.write_examples(write, command)
if notes:
self.write_heading(write, "NOTES")
write(notes)
self.write_see_also(write, man_title)
self.gen_man_page(man_title, doc.getvalue())
# Generate the borg-common(1) man page with the common options.
if "create" in choices:
doc, write = self.new_doc()
man_title = "borg-common"
self.write_man_header(write, man_title, "Common options of Borg commands")
common_options = [group for group in choices["create"]._action_groups if group.title == "Common options"][0]
self.write_heading(write, "SYNOPSIS")
self.write_options_group(write, common_options)
self.write_see_also(write, man_title)
self.gen_man_page(man_title, doc.getvalue())
return is_subcommand
def build_topic_pages(self, Archiver):
for topic, text in Archiver.helptext.items():
doc, write = self.new_doc()
man_title = "borg-" + topic
print("building man page", man_title + "(1)", file=sys.stderr)
self.write_man_header(write, man_title, "Details regarding " + topic)
self.write_heading(write, "DESCRIPTION")
write(text)
self.gen_man_page(man_title, doc.getvalue())
def build_intro_page(self):
doc, write = self.new_doc()
man_title = "borg"
print("building man page borg(1)", file=sys.stderr)
with open("docs/man_intro.rst") as fd:
man_intro = fd.read()
self.write_man_header(write, man_title, "deduplicating and encrypting backup tool")
self.gen_man_page(man_title, doc.getvalue() + man_intro)
def new_doc(self):
doc = io.StringIO(self.rst_prelude)
doc.read()
write = self.printer(doc)
return doc, write
def printer(self, fd):
def write(*args, **kwargs):
print(*args, file=fd, **kwargs)
return write
def write_heading(self, write, header, char="-", double_sided=False):
write()
if double_sided:
write(char * len(header))
write(header)
write(char * len(header))
write()
def write_man_header(self, write, title, description):
self.write_heading(write, title, "=", double_sided=True)
self.write_heading(write, description, double_sided=True)
# man page metadata
write(":Author: The Borg Collective")
source_date_epoch = int(os.environ.get("SOURCE_DATE_EPOCH", time.time()))
write(":Date:", datetime.fromtimestamp(source_date_epoch, timezone.utc).date().isoformat())
write(":Manual section: 1")
write(":Manual group: borg backup tool")
write()
def write_examples(self, write, command):
command = command.replace(" ", "_")
with open("docs/usage/%s.rst" % self.usage_group.get(command, command)) as fd:
usage = fd.read()
usage_include = ".. include:: %s.rst.inc" % command
begin = usage.find(usage_include)
end = usage.find(".. include", begin + 1)
# If a command has a dedicated anchor, it will occur before the command's include.
if 0 < usage.find(".. _", begin + 1) < end:
end = usage.find(".. _", begin + 1)
examples = usage[begin:end]
examples = examples.replace(usage_include, "")
examples = examples.replace("Examples\n~~~~~~~~", "")
examples = examples.replace("Miscellaneous Help\n------------------", "")
examples = examples.replace("``docs/misc/prune-example.txt``:", "``docs/misc/prune-example.txt``.")
examples = examples.replace(".. highlight:: none\n", "") # we don't support highlight
examples = re.sub("^(~+)$", lambda matches: "+" * len(matches.group(0)), examples, flags=re.MULTILINE)
examples = examples.strip()
if examples:
self.write_heading(write, "EXAMPLES", "-")
write(examples)
def write_see_also(self, write, man_title):
see_also = self.see_also.get(man_title.replace("borg-", ""), ())
see_also = ["`borg-%s(1)`" % s for s in see_also]
see_also.insert(0, "`borg-common(1)`")
self.write_heading(write, "SEE ALSO")
write(", ".join(see_also))
def gen_man_page(self, name, rst):
from docutils.writers import manpage
from docutils.core import publish_string
from docutils.nodes import inline
from docutils.parsers.rst import roles
def issue(name, rawtext, text, lineno, inliner, options={}, content=[]):
return [inline(rawtext, "#" + text)], []
roles.register_local_role("issue", issue)
# We give the source_path so that docutils can find relative includes
# as-if the document where located in the docs/ directory.
man_page = publish_string(source=rst, source_path="docs/%s.rst" % name, writer=manpage.Writer())
with open("docs/man/%s.1" % name, "wb") as fd:
fd.write(man_page)
def write_usage(self, write, parser):
if any(len(o.option_strings) for o in parser._actions):
write(" [options] ", end="")
for option in parser._actions:
if option.option_strings:
continue
write(format_metavar(option), end=" ")
def write_options(self, write, parser):
for group in parser._action_groups:
if group.title == "Common options" or not group._group_actions:
continue
title = "arguments" if group.title == "positional arguments" else group.title
self.write_heading(write, title, "+")
self.write_options_group(write, group)
def write_options_group(self, write, group):
def is_positional_group(group):
return any(not o.option_strings for o in group._group_actions)
if is_positional_group(group):
for option in group._group_actions:
write(option.metavar)
write(textwrap.indent(option.help or "", " " * 4))
return
opts = OrderedDict()
for option in group._group_actions:
if option.metavar:
option_fmt = "%s " + option.metavar
else:
option_fmt = "%s"
option_str = ", ".join(option_fmt % s for s in option.option_strings)
option_desc = textwrap.dedent((option.help or "") % option.__dict__)
opts[option_str] = textwrap.indent(option_desc, " " * 4)
padding = len(max(opts)) + 1
for option, desc in opts.items():
write(option.ljust(padding), desc)
cython_sources = """
src/borg/compress.pyx
src/borg/crypto/low_level.pyx
src/borg/chunker.pyx
src/borg/hashindex.pyx
src/borg/item.pyx
src/borg/checksums.pyx
src/borg/platform/posix.pyx
src/borg/platform/linux.pyx
src/borg/platform/syncfilerange.pyx
src/borg/platform/darwin.pyx
src/borg/platform/freebsd.pyx
src/borg/platform/windows.pyx
""".strip().splitlines()
def rm(file):
try:
os.unlink(file)
except FileNotFoundError:
return False
else:
return True
class Clean:
def run(self):
for source in cython_sources:
genc = source.replace(".pyx", ".c")
rm(genc)
compiled_glob = source.replace(".pyx", ".cpython*")
for compiled in sorted(glob.glob(compiled_glob)):
rm(compiled)
return 0
def usage():
print(
textwrap.dedent(
"""
Usage:
python scripts/make.py clean # clean workdir (remove generated files)
python scripts/make.py build_usage # build usage documentation
python scripts/make.py build_man # build man pages
"""
)
)
def main(argv):
if len(argv) < 2 or len(argv) == 2 and argv[1] in ("-h", "--help"):
usage()
return 0
command = argv[1]
if command == "clean":
return Clean().run()
if command == "build_usage":
return BuildUsage().run()
if command == "build_man":
return BuildMan().run()
usage()
return 1
if __name__ == "__main__":
rc = main(sys.argv)
sys.exit(rc)