f4pga: fix on-demand products requiring being demanded (#606)

f4pga/__init__.py

@@ -327,8 +327,9 @@ class Flow:
 
         outputs = module_map(provider.module, modrunctx)
         for output_paths in outputs.values():
-            if req_exists(output_paths) and self.f4cache:
-                _cache_deps(output_paths, self.f4cache)
+            if output_paths is not None:
+                if req_exists(output_paths) and self.f4cache:
+                    _cache_deps(output_paths, self.f4cache)
 
         stages_checked.add(provider.name)
         self.dep_paths.update(outputs)
@@ -434,11 +435,11 @@ class Flow:
             self.run_stages.discard(provider.name)
 
             for product in provider.produces:
-                exists = req_exists(paths)
-                if (product.spec == 'req') and not exists:
+                if (product.spec == 'req') and not req_exists(paths):
                     raise DependencyNotProducedException(dep, provider.name)
-                if exists and self.f4cache:
-                    _cache_deps(self.dep_paths[product.name], self.f4cache)
+                prod_paths = self.dep_paths[product.name]
+                if (prod_paths is not None) and req_exists(paths) and self.f4cache:
+                    _cache_deps(prod_paths, self.f4cache)
 
         return True
 

f4pga/common.py

@@ -94,17 +94,20 @@ def resolve_modstr(modstr: str):
     return str(Path(col_path) / module_filename)
 
 
-def deep(fun):
+def deep(fun, allow_none=False):
     """
     Create a recursive string transform function for 'str | list | dict', i.e a dependency.
     """
     def d(paths, *args, **kwargs):
+        nonlocal allow_none
         if type(paths) is str:
             return fun(paths, *args, **kwargs)
         elif type(paths) is list:
             return [d(p, *args, **kwargs) for p in paths];
         elif type(paths) is dict:
             return dict([(k, d(p, *args, **kwargs)) for k, p in paths.items()])
+        elif allow_none and (paths is None):
+            return paths
         else:
             raise RuntimeError(f'paths is of type {type(paths)}')
     return d

f4pga/module_runner.py

@@ -109,6 +109,8 @@ def module_io(module: Module):
     }
 
 
+_deep_resolve = deep(lambda p: str(Path(p).resolve()), allow_none=True)
+
 def module_map(module: Module, ctx: ModRunCtx):
     try:
         mod_ctx = ModuleContext(
@@ -121,7 +123,7 @@ def module_map(module: Module, ctx: ModRunCtx):
     except Exception as e:
         raise ModuleFailException(module.name, 'map', e)
 
-    return deep(lambda p: str(Path(p).resolve()))(vars(mod_ctx.outputs))
+    return _deep_resolve(vars(mod_ctx.outputs))
 
 
 def module_exec(module: Module, ctx: ModRunCtx):