Fix on-demand dependencies requiring being demanded

Signed-off-by: Krzysztof Boronski <kboronski@antmicro.com>
This commit is contained in:
Krzysztof Boronski 2022-08-05 12:27:51 -05:00
parent e9a520a17a
commit 8ed3a5bd58
3 changed files with 14 additions and 8 deletions

View File

@ -327,8 +327,9 @@ class Flow:
outputs = module_map(provider.module, modrunctx)
for output_paths in outputs.values():
if req_exists(output_paths) and self.f4cache:
_cache_deps(output_paths, self.f4cache)
if output_paths is not None:
if req_exists(output_paths) and self.f4cache:
_cache_deps(output_paths, self.f4cache)
stages_checked.add(provider.name)
self.dep_paths.update(outputs)
@ -434,11 +435,11 @@ class Flow:
self.run_stages.discard(provider.name)
for product in provider.produces:
exists = req_exists(paths)
if (product.spec == 'req') and not exists:
if (product.spec == 'req') and not req_exists(paths):
raise DependencyNotProducedException(dep, provider.name)
if exists and self.f4cache:
_cache_deps(self.dep_paths[product.name], self.f4cache)
prod_paths = self.dep_paths[product.name]
if (prod_paths is not None) and req_exists(paths) and self.f4cache:
_cache_deps(prod_paths, self.f4cache)
return True

View File

@ -94,17 +94,20 @@ def resolve_modstr(modstr: str):
return str(Path(col_path) / module_filename)
def deep(fun):
def deep(fun, allow_none=False):
"""
Create a recursive string transform function for 'str | list | dict', i.e a dependency.
"""
def d(paths, *args, **kwargs):
nonlocal allow_none
if type(paths) is str:
return fun(paths, *args, **kwargs)
elif type(paths) is list:
return [d(p, *args, **kwargs) for p in paths];
elif type(paths) is dict:
return dict([(k, d(p, *args, **kwargs)) for k, p in paths.items()])
elif allow_none and (paths is None):
return paths
else:
raise RuntimeError(f'paths is of type {type(paths)}')
return d

View File

@ -109,6 +109,8 @@ def module_io(module: Module):
}
_deep_resolve = deep(lambda p: str(Path(p).resolve()), allow_none=True)
def module_map(module: Module, ctx: ModRunCtx):
try:
mod_ctx = ModuleContext(
@ -121,7 +123,7 @@ def module_map(module: Module, ctx: ModRunCtx):
except Exception as e:
raise ModuleFailException(module.name, 'map', e)
return deep(lambda p: str(Path(p).resolve()))(vars(mod_ctx.outputs))
return _deep_resolve(vars(mod_ctx.outputs))
def module_exec(module: Module, ctx: ModRunCtx):