conans/client/graph/graph_builder.py
import time
from conans.client.conanfile.configure import run_configure_method
from conans.client.graph.graph import DepsGraph, Node, RECIPE_EDITABLE, CONTEXT_HOST, CONTEXT_BUILD
from conans.errors import (ConanException, ConanExceptionInUserConanfileMethod,
conanfile_exception_formatter, ConanInvalidConfiguration)
from conans.model.conan_file import get_env_context_manager
from conans.model.ref import ConanFileReference
from conans.model.requires import Requirements, Requirement
from conans.util.log import logger
class DepsGraphBuilder(object):
"""
This is a summary, in pseudo-code of the execution and structure of the graph
resolution algorithm
load_graph(root_node)
init root_node
expand_node(root_node)
# 1. Evaluate requirements(), overrides, and version ranges
get_node_requirements(node)
node.conanfile.requirements() # call the requirements()
resolve_cached_alias(node.conanfile.requires) # replace cached alias
update_requires_from_downstream(down_requires) # process overrides
resolve_ranges(node) # resolve version-ranges
resolve_cached_alias(node.conanfile.requires) # replace cached alias again
# 2. Process each requires of this node
for req in node.conanfile.requires:
expand_require(req)
if req.name not in graph: # New node
new_node = create_new_node(req) # fetch and load conanfile.py
if alias => create_new_node(alias) # recurse alias
expand_node(new_node) # recursion
else: # Node exists, closing diamond
resolve_cached_alias(req)
check_conflicts(req) # diamonds can cause conflicts
if need_recurse: # check for conflicts upstream
expand_node(previous_node) # recursion
"""
def __init__(self, proxy, output, loader, resolver, recorder):
self._proxy = proxy
self._output = output
self._loader = loader
self._resolver = resolver
self._recorder = recorder
def load_graph(self, root_node, check_updates, update, remotes, profile_host, profile_build,
graph_lock=None):
check_updates = check_updates or update
initial = graph_lock.initial_counter if graph_lock else None
dep_graph = DepsGraph(initial_node_id=initial)
# compute the conanfile entry point for this dependency graph
root_node.public_closure.add(root_node)
root_node.public_deps.add(root_node)
root_node.transitive_closure[root_node.name] = root_node
if profile_build:
root_node.conanfile.settings_build = profile_build.processed_settings.copy()
root_node.conanfile.settings_target = None
dep_graph.add_node(root_node)
# enter recursive computation
t1 = time.time()
self._expand_node(root_node, dep_graph, Requirements(), None, None, check_updates,
update, remotes, profile_host, profile_build, graph_lock)
logger.debug("GRAPH: Time to load deps %s" % (time.time() - t1))
return dep_graph
def extend_build_requires(self, graph, node, build_requires_refs, check_updates, update,
remotes, profile_host, profile_build, graph_lock):
# The options that will be defined in the node will be the real options values that have
# been already propagated downstream from the dependency graph. This will override any
# other possible option in the build_requires dependency graph. This means that in theory
# an option conflict while expanding the build_requires is impossible
node.conanfile.build_requires_options.clear_unscoped_options()
new_options = node.conanfile.build_requires_options._reqs_options
# FIXME: Convert to pattern-based for build_requires, do not fail hard if some option
# was removed
if profile_build: # Apply the fix only for the 2 profile scenario
new_options = {k+"*": v for k, v in new_options.items()}
new_reqs = Requirements()
conanfile = node.conanfile
scope = conanfile.display_name
build_requires = []
for ref, context in build_requires_refs:
r = Requirement(ref)
r.build_require = True
r.build_require_context = context
r.force_host_context = getattr(ref, "force_host_context", False)
build_requires.append(r)
if graph_lock:
graph_lock.pre_lock_node(node)
# TODO: Add info about context?
graph_lock.lock_node(node, build_requires, build_requires=True)
for require in build_requires:
self._resolve_alias(node, require, graph, update, update, remotes)
self._resolve_ranges(graph, build_requires, scope, update, remotes)
for br in build_requires:
context_switch = bool(br.build_require_context == CONTEXT_BUILD)
populate_settings_target = context_switch # Avoid 'settings_target' for BR-host
self._expand_require(br, node, graph, check_updates, update,
remotes, profile_host, profile_build, new_reqs, new_options,
graph_lock, context_switch=context_switch,
populate_settings_target=populate_settings_target)
new_nodes = set(n for n in graph.nodes if n.package_id is None)
# This is to make sure that build_requires have precedence over the normal requires
node.public_closure.sort(key_fn=lambda x: x not in new_nodes)
return new_nodes
def _expand_node(self, node, graph, down_reqs, down_ref, down_options, check_updates, update,
remotes, profile_host, profile_build, graph_lock):
""" expands the dependencies of the node, recursively
param node: Node object to be expanded in this step
down_reqs: the Requirements as coming from downstream, which can overwrite current
values
param down_ref: ConanFileReference of who is depending on current node for this expansion
"""
# basic node configuration: calling configure() and requirements() and version-ranges
new_options, new_reqs = self._get_node_requirements(node, graph, down_ref, down_options,
down_reqs, graph_lock, update, remotes)
# Expand each one of the current requirements
for require in node.conanfile.requires.values():
if require.override:
continue
self._expand_require(require, node, graph, check_updates, update, remotes, profile_host,
profile_build, new_reqs, new_options, graph_lock,
context_switch=False)
def _resolve_ranges(self, graph, requires, consumer, update, remotes):
for require in requires:
if require.locked_id: # if it is locked, nothing to resolved
continue
self._resolver.resolve(require, consumer, update, remotes)
self._resolve_cached_alias(requires, graph)
@staticmethod
def _resolve_cached_alias(requires, graph):
if graph.aliased:
for require in requires:
alias = graph.aliased.get(require.ref)
if alias:
require.ref = alias
def _resolve_alias(self, node, require, graph, check_updates, update, remotes):
alias = require.alias
if alias is None:
return
# First try cached
cached = graph.new_aliased.get(alias)
if cached is not None:
while True:
new_cached = graph.new_aliased.get(cached)
if new_cached is None:
break
else:
cached = new_cached
require.ref = cached
return
while alias is not None:
# if not cached, then resolve
try:
result = self._proxy.get_recipe(alias, check_updates, update, remotes, self._recorder)
conanfile_path, recipe_status, remote, new_ref = result
except ConanException as e:
raise e
dep_conanfile = self._loader.load_basic(conanfile_path)
try:
pointed_ref = ConanFileReference.loads(dep_conanfile.alias)
except Exception as e:
raise ConanException("Alias definition error in {}: {}".format(alias, str(e)))
# UPDATE THE REQUIREMENT!
require.ref = require.range_ref = pointed_ref
graph.new_aliased[alias] = pointed_ref # Caching the alias
new_req = Requirement(pointed_ref) # FIXME: Ugly temp creation just for alias check
alias = new_req.alias
def _get_node_requirements(self, node, graph, down_ref, down_options, down_reqs, graph_lock,
update, remotes):
""" compute the requirements of a node, evaluating requirements(), propagating
the downstream requirements and overrides and solving version-ranges
"""
# basic node configuration: calling configure() and requirements()
if graph_lock:
graph_lock.pre_lock_node(node)
new_options = self._config_node(node, down_ref, down_options)
for require in node.conanfile.requires.values():
self._resolve_alias(node, require, graph, update, update, remotes)
# Alias that are cached should be replaced here, bc next requires.update() will warn if not
self._resolve_cached_alias(node.conanfile.requires.values(), graph)
if graph_lock: # No need to evaluate, they are hardcoded in lockfile
graph_lock.lock_node(node, node.conanfile.requires.values())
# propagation of requirements can be necessary if some nodes are not locked
new_reqs = node.conanfile.requires.update(down_reqs, self._output, node.ref, down_ref)
# if there are version-ranges, resolve them before expanding each of the requirements
# Resolve possible version ranges of the current node requirements
# new_reqs is a shallow copy of what is propagated upstream, so changes done by the
# RangeResolver are also done in new_reqs, and then propagated!
conanfile = node.conanfile
scope = conanfile.display_name
self._resolve_ranges(graph, conanfile.requires.values(), scope, update, remotes)
if not hasattr(conanfile, "_conan_evaluated_requires"):
conanfile._conan_evaluated_requires = conanfile.requires.copy()
elif conanfile.requires != conanfile._conan_evaluated_requires:
raise ConanException("%s: Incompatible requirements obtained in different "
"evaluations of 'requirements'\n"
" Previous requirements: %s\n"
" New requirements: %s"
% (scope, list(conanfile._conan_evaluated_requires.values()),
list(conanfile.requires.values())))
return new_options, new_reqs
def _expand_require(self, require, node, graph, check_updates, update, remotes, profile_host,
profile_build, new_reqs, new_options, graph_lock, context_switch,
populate_settings_target=True):
# Handle a requirement of a node. There are 2 possibilities
# node -(require)-> new_node (creates a new node in the graph)
# node -(require)-> previous (creates a diamond with a previously existing node)
# If the required is found in the node ancestors a loop is being closed
context = CONTEXT_BUILD if context_switch else node.context
name = require.ref.name # TODO: allow bootstrapping, use references instead of names
if node.ancestors.get(name, context) or (name == node.name and context == node.context):
raise ConanException("Loop detected in context %s: '%s' requires '%s'"
" which is an ancestor too" % (context, node.ref, require.ref))
# If the requirement is found in the node public dependencies, it is a diamond
previous = node.public_deps.get(name, context=context)
previous_closure = node.public_closure.get(name, context=context)
# build_requires and private will create a new node if it is not in the current closure
if not previous or ((require.build_require or require.private) and not previous_closure):
# new node, must be added and expanded (node -> new_node)
new_node = self._create_new_node(node, graph, require, check_updates, update,
remotes, profile_host, profile_build, graph_lock,
context_switch=context_switch,
populate_settings_target=populate_settings_target)
# The closure of a new node starts with just itself
new_node.public_closure.add(new_node)
new_node.transitive_closure[new_node.name] = new_node
# The new created node is connected to the parent one
node.connect_closure(new_node)
if require.private or require.build_require:
# If the requirement is private (or build_require), a new public_deps is defined
# the new_node doesn't propagate downstream the "node" consumer, so its public_deps
# will be a copy of the node.public_closure, i.e. it can only cause conflicts in the
# new_node.public_closure.
new_node.public_deps.assign(node.public_closure)
new_node.public_deps.add(new_node)
else:
node.transitive_closure[new_node.name] = new_node
# Normal requires propagate and can conflict with the parent "node.public_deps" too
new_node.public_deps.assign(node.public_deps)
new_node.public_deps.add(new_node)
# All the dependents of "node" are also connected now to "new_node"
for dep_node in node.inverse_closure:
dep_node.connect_closure(new_node)
# RECURSION, keep expanding (depth-first) the new node
self._expand_node(new_node, graph, new_reqs, node.ref, new_options, check_updates,
update, remotes, profile_host, profile_build, graph_lock)
if not require.private and not require.build_require:
for name, n in new_node.transitive_closure.items():
node.transitive_closure[name] = n
else: # a public node already exist with this name
self._resolve_cached_alias([require], graph)
# As we are closing a diamond, there can be conflicts. This will raise if conflicts
conflict = self._conflicting_references(previous, require.ref, node.ref)
if conflict: # It is possible to get conflict from alias, try to resolve it
self._resolve_recipe(node, graph, require, check_updates,
update, remotes, profile_host, graph_lock)
# Maybe it was an ALIAS, so we can check conflict again
conflict = self._conflicting_references(previous, require.ref, node.ref)
if conflict:
raise ConanException(conflict)
# Add current ancestors to the previous node and upstream deps
for n in previous.public_closure:
n.ancestors.add(node)
for item in node.ancestors:
n.ancestors.add(item)
node.connect_closure(previous)
graph.add_edge(node, previous, require)
if not require.private and not require.build_require:
for name, n in previous.transitive_closure.items():
node.transitive_closure[name] = n
# All the upstream dependencies (public_closure) of the previously existing node
# now will be also connected to the node and to all its dependants
for n in previous.transitive_closure.values():
node.connect_closure(n)
for dep_node in node.inverse_closure:
dep_node.connect_closure(n)
# Recursion is only necessary if the inputs conflict with the current "previous"
# configuration of upstream versions and options
# recursion can stop if there is a graph_lock not relaxed
lock_recurse = not (graph_lock and not graph_lock.relaxed)
if lock_recurse and self._recurse(previous.public_closure, new_reqs, new_options,
previous.context):
self._expand_node(previous, graph, new_reqs, node.ref, new_options, check_updates,
update, remotes, profile_host, profile_build, graph_lock)
@staticmethod
def _conflicting_references(previous, new_ref, consumer_ref=None):
if previous.ref.copy_clear_rev() != new_ref.copy_clear_rev():
if consumer_ref:
return ("Conflict in %s:\n"
" '%s' requires '%s' while '%s' requires '%s'.\n"
" To fix this conflict you need to override the package '%s' "
"in your root package."
% (consumer_ref, consumer_ref, new_ref, next(iter(previous.dependants)).src,
previous.ref, new_ref.name))
return "Unresolvable conflict between {} and {}".format(previous.ref, new_ref)
# Computed node, if is Editable, has revision=None
# If new_ref.revision is None we cannot assume any conflict, the user hasn't specified
# a revision, so it's ok any previous_ref
if previous.ref.revision and new_ref.revision and previous.ref.revision != new_ref.revision:
if consumer_ref:
raise ConanException("Conflict in %s\n"
" Different revisions of %s has been requested"
% (consumer_ref, new_ref))
return True
return False
def _recurse(self, closure, new_reqs, new_options, context):
""" For a given closure, if some requirements or options coming from downstream
is incompatible with the current closure, then it is necessary to recurse
then, incompatibilities will be raised as usually"""
for req in new_reqs.values():
n = closure.get(req.ref.name, context=context)
if n and self._conflicting_references(n, req.ref):
return True
for pkg_name, options_values in new_options.items():
n = closure.get(pkg_name, context=context)
if n:
options = n.conanfile.options
for option, value in options_values.items():
if getattr(options, option) != value:
return True
return False
@staticmethod
def _config_node(node, down_ref, down_options):
""" update settings and option in the current ConanFile, computing actual
requirement values, cause they can be overridden by downstream requires
param settings: dict of settings values => {"os": "windows"}
"""
conanfile, ref = node.conanfile, node.ref
try:
run_configure_method(conanfile, down_options, down_ref, ref)
with get_env_context_manager(conanfile, without_python=True):
# Update requirements (overwrites), computing new upstream
if hasattr(conanfile, "requirements"):
# If re-evaluating the recipe, in a diamond graph, with different options,
# it could happen that one execution path of requirements() defines a package
# and another one a different package raising Duplicate dependency error
# Or the two consecutive calls, adding 2 different dependencies for the 2 paths
# So it is necessary to save the "requires" state and restore it before a second
# execution of requirements(). It is a shallow copy, if first iteration is
# RequireResolve'd or overridden, the inner requirements are modified
if not hasattr(conanfile, "_conan_original_requires"):
conanfile._conan_original_requires = conanfile.requires.copy()
else:
conanfile.requires = conanfile._conan_original_requires.copy()
with conanfile_exception_formatter(str(conanfile), "requirements"):
conanfile.requirements()
new_options = conanfile.options.deps_package_values
except ConanExceptionInUserConanfileMethod:
raise
except ConanException as e:
raise ConanException("%s: %s" % (ref or "Conanfile", str(e)))
except Exception as e:
raise ConanException(e)
return new_options
def _resolve_recipe(self, current_node, dep_graph, requirement, check_updates,
update, remotes, profile, graph_lock, original_ref=None):
ref = requirement.ref
ref_version = str(ref.version)
if ref_version.startswith("<host_version") and ref_version.endswith(">"):
if not requirement.build_require:
raise ConanException(f"{current_node.ref} uses '<host_version>' in requires, "
"it is only allowed in tool_requires")
search_ref_name = ref.name
tracking_ref = ref_version.split(":", 1)
if len(tracking_ref) == 2:
search_ref_name = tracking_ref[1][:-1] # Remove trailing '>'
transitive = next((dep for dep in current_node.dependencies if
not dep.build_require and
dep.require.ref.name == search_ref_name and
dep.require.ref.user == ref.user and
dep.require.ref.channel == ref.channel), None)
if transitive is None:
raise ConanException(
f"{current_node.ref} require '{search_ref_name}/{ref_version}': didn't find a matching host dependency")
new_ref = transitive.require.ref
# Final ref is the one from the transitive dependency, but with the version from the
# tracked requirement, if any
requirement.ref = ConanFileReference(ref.name, new_ref.version,
new_ref.user, new_ref.channel)
try:
result = self._proxy.get_recipe(requirement.ref, check_updates, update,
remotes, self._recorder)
except ConanException as e:
if current_node.ref:
self._output.error("Failed requirement '%s' from '%s'"
% (requirement.ref, current_node.conanfile.display_name))
raise e
conanfile_path, recipe_status, remote, new_ref = result
locked_id = requirement.locked_id
lock_py_requires = graph_lock.python_requires(locked_id) if locked_id is not None else None
dep_conanfile = self._loader.load_conanfile(conanfile_path, profile, ref=requirement.ref,
lock_python_requires=lock_py_requires)
if recipe_status == RECIPE_EDITABLE:
dep_conanfile.in_local_cache = False
dep_conanfile.develop = True
if getattr(dep_conanfile, "alias", None):
new_ref_norev = new_ref.copy_clear_rev()
pointed_ref = ConanFileReference.loads(dep_conanfile.alias)
dep_graph.aliased[new_ref_norev] = pointed_ref # Caching the alias
requirement.ref = pointed_ref
if original_ref: # So transitive alias resolve to the latest in the chain
dep_graph.aliased[original_ref] = pointed_ref
return self._resolve_recipe(current_node, dep_graph, requirement, check_updates,
update, remotes, profile, graph_lock, original_ref)
return new_ref, dep_conanfile, recipe_status, remote, locked_id
def _create_new_node(self, current_node, dep_graph, requirement, check_updates,
update, remotes, profile_host, profile_build, graph_lock, context_switch,
populate_settings_target):
# If there is a context_switch, it is because it is a BR-build
if context_switch:
profile = profile_build
context = CONTEXT_BUILD
else:
profile = profile_host if current_node.context == CONTEXT_HOST else profile_build
context = current_node.context
result = self._resolve_recipe(current_node, dep_graph, requirement, check_updates, update,
remotes, profile, graph_lock)
new_ref, dep_conanfile, recipe_status, remote, locked_id = result
# Assign the profiles depending on the context
if profile_build: # Keep existing behavior (and conanfile members) if no profile_build
dep_conanfile.settings_build = profile_build.processed_settings.copy()
if not context_switch:
if populate_settings_target:
dep_conanfile.settings_target = current_node.conanfile.settings_target
else:
dep_conanfile.settings_target = None
else:
if current_node.context == CONTEXT_HOST:
dep_conanfile.settings_target = profile_host.processed_settings.copy()
else:
dep_conanfile.settings_target = profile_build.processed_settings.copy()
logger.debug("GRAPH: new_node: %s" % str(new_ref))
new_node = Node(new_ref, dep_conanfile, context=context)
new_node.revision_pinned = requirement.ref.revision is not None
new_node.recipe = recipe_status
new_node.remote = remote
# Ancestors are a copy of the parent, plus the parent itself
new_node.ancestors.assign(current_node.ancestors)
new_node.ancestors.add(current_node)
if locked_id is not None:
new_node.id = locked_id
dep_graph.add_node(new_node)
dep_graph.add_edge(current_node, new_node, requirement)
return new_node