f4pga: merge sfbuild into __init__
Signed-off-by: Unai Martinez-Corral <umartinezcorral@antmicro.com>
This commit is contained in:
Unai Martinez-Corral
parent
920d1d5ec3
commit
fcb4ce8811
|
|
@ -55,5 +55,5 @@ jobs:
|
||||||
|
|
||||||
- name: Test py4FPGA (PYTHONPATH)
|
- name: Test py4FPGA (PYTHONPATH)
|
||||||
run: |
|
run: |
|
||||||
PYTHONPATH=$(pwd) python3 f4pga/sfbuild.py
|
PYTHONPATH=$(pwd) python3 f4pga/__init__.py
|
||||||
PYTHONPATH=$(pwd) python3 f4pga/sfbuild.py -h
|
PYTHONPATH=$(pwd) python3 f4pga/__init__.py -h
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,645 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
"""
|
||||||
|
sfbuild - Symbiflow Build System
|
||||||
|
|
||||||
|
This tool allows for building FPGA targets (such as bitstreams) for any supported
|
||||||
|
platform with just one simple command and a project file.
|
||||||
|
|
||||||
|
The idea is that sfbuild wraps all the tools needed by different platforms in
|
||||||
|
"modules", which define inputs/outputs and various parameters. This allows
|
||||||
|
sfbuild to resolve dependencies for any target provided that a "flow definition"
|
||||||
|
file exists for such target. The flow defeinition file list modules available for
|
||||||
|
that platform and may tweak some settings of those modules.
|
||||||
|
|
||||||
|
A basic example of using sfbuild:
|
||||||
|
$ sfbuild build --platform arty_35 -t bitstream
|
||||||
|
|
||||||
|
This will make sfbuild attempt to create a bitstream for arty_35 platform.
|
||||||
|
flow.json is a flow configuration file, which should be created for a project
|
||||||
|
that uses sfbuild. Iontains project-specific definitions needed within the flow,
|
||||||
|
such as list of source code files.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
from argparse import Namespace
|
||||||
|
import os
|
||||||
|
from os import environ
|
||||||
|
import json
|
||||||
|
from typing import Iterable
|
||||||
|
from colorama import Fore, Style
|
||||||
|
from f4pga.sf_common import ResolutionEnv, fatal, scan_modules, set_verbosity_level, \
|
||||||
|
sfprint
|
||||||
|
from f4pga.sf_module import *
|
||||||
|
from f4pga.sf_cache import SymbiCache
|
||||||
|
import f4pga.sf_ugly as sf_ugly
|
||||||
|
from f4pga.sf_flow_config import ProjectFlowConfig, FlowConfig, FlowDefinition, \
|
||||||
|
open_project_flow_cfg, verify_platform_name, \
|
||||||
|
verify_stage
|
||||||
|
from f4pga.sf_module_runner import *
|
||||||
|
from f4pga.sf_module_inspector import get_module_info
|
||||||
|
from f4pga.sf_stage import Stage
|
||||||
|
from f4pga.sf_argparse import setup_argparser, get_cli_flow_config
|
||||||
|
|
||||||
|
SYMBICACHEPATH = '.symbicache'
|
||||||
|
|
||||||
|
binpath = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(os.sys.argv[0])), '..'))
|
||||||
|
mypath = str(Path(__file__).resolve().parent)
|
||||||
|
|
||||||
|
share_dir_path = os.path.realpath(f"{environ.get('INSTALL_DIR', '/usr/local')}/xc7/install/share/symbiflow")
|
||||||
|
|
||||||
|
class DependencyNotProducedException(Exception):
|
||||||
|
dep_name: str
|
||||||
|
provider: str
|
||||||
|
|
||||||
|
def __init__(self, dep_name: str, provider: str):
|
||||||
|
self.dep_name = dep_name
|
||||||
|
self.provider = provider
|
||||||
|
|
||||||
|
def __str__(self) -> str:
|
||||||
|
return f'Stage `{self.provider}` did not produce promised ' \
|
||||||
|
f'dependency `{self.dep_name}`'
|
||||||
|
|
||||||
|
def dep_value_str(dep: str):
|
||||||
|
return ':' + dep
|
||||||
|
|
||||||
|
def platform_stages(platform_flow, r_env):
|
||||||
|
""" Iterates over all stages available in a given flow. """
|
||||||
|
|
||||||
|
stage_options = platform_flow.get('stage_options')
|
||||||
|
for stage_name, modulestr in platform_flow['stages'].items():
|
||||||
|
mod_opts = stage_options.get(stage_name) if stage_options else None
|
||||||
|
yield Stage(stage_name, modulestr, mod_opts, r_env)
|
||||||
|
|
||||||
|
def req_exists(r):
|
||||||
|
""" Checks whether a dependency exists on a drive. """
|
||||||
|
|
||||||
|
if type(r) is str:
|
||||||
|
if not os.path.isfile(r) and not os.path.islink(r) \
|
||||||
|
and not os.path.isdir(r):
|
||||||
|
return False
|
||||||
|
elif type(r) is list:
|
||||||
|
return not (False in map(req_exists, r))
|
||||||
|
else:
|
||||||
|
raise Exception(f'Requirements can be currently checked only for single '
|
||||||
|
f'paths, or path lists (reason: {r})')
|
||||||
|
return True
|
||||||
|
|
||||||
|
def map_outputs_to_stages(stages: 'list[Stage]'):
|
||||||
|
"""
|
||||||
|
Associates a stage with every possible output.
|
||||||
|
This is commonly refferef to as `os_map` (output-stage-map) through the code.
|
||||||
|
"""
|
||||||
|
|
||||||
|
os_map: 'dict[str, Stage]' = {} # Output-Stage map
|
||||||
|
for stage in stages:
|
||||||
|
for output in stage.produces:
|
||||||
|
if not os_map.get(output.name):
|
||||||
|
os_map[output.name] = stage
|
||||||
|
elif os_map[output.name] != stage:
|
||||||
|
raise Exception(f'Dependency `{output.name}` is generated by '
|
||||||
|
f'stage `{os_map[output.name].name}` and '
|
||||||
|
f'`{stage.name}`. Dependencies can have only one '
|
||||||
|
'provider at most.')
|
||||||
|
return os_map
|
||||||
|
|
||||||
|
def filter_existing_deps(deps: 'dict[str, ]', symbicache):
|
||||||
|
return [(n, p) for n, p in deps.items() \
|
||||||
|
if req_exists(p)] # and not dep_differ(p, symbicache)]
|
||||||
|
|
||||||
|
def get_stage_values_override(og_values: dict, stage: Stage):
|
||||||
|
values = og_values.copy()
|
||||||
|
values.update(stage.value_ovds)
|
||||||
|
return values
|
||||||
|
|
||||||
|
def prepare_stage_io_input(stage: Stage):
|
||||||
|
return { 'params': stage.params } if stage.params is not None else {}
|
||||||
|
|
||||||
|
def prepare_stage_input(stage: Stage, platform_name: str, values: dict,
|
||||||
|
dep_paths: 'dict[str, ]', config_paths: 'dict[str, ]'):
|
||||||
|
takes = {}
|
||||||
|
for take in stage.takes:
|
||||||
|
paths = dep_paths.get(take.name)
|
||||||
|
if paths: # Some takes may be not required
|
||||||
|
takes[take.name] = paths
|
||||||
|
|
||||||
|
produces = {}
|
||||||
|
for prod in stage.produces:
|
||||||
|
if dep_paths.get(prod.name):
|
||||||
|
produces[prod.name] = dep_paths[prod.name]
|
||||||
|
elif config_paths.get(prod.name):
|
||||||
|
produces[prod.name] = config_paths[prod.name]
|
||||||
|
|
||||||
|
stage_mod_cfg = {
|
||||||
|
'takes': takes,
|
||||||
|
'produces': produces,
|
||||||
|
'values': values,
|
||||||
|
'platform': platform_name,
|
||||||
|
}
|
||||||
|
|
||||||
|
return stage_mod_cfg
|
||||||
|
|
||||||
|
def update_dep_statuses(paths, consumer: str, symbicache: SymbiCache):
|
||||||
|
if type(paths) is str:
|
||||||
|
return symbicache.update(paths, consumer)
|
||||||
|
elif type(paths) is list:
|
||||||
|
for p in paths:
|
||||||
|
return update_dep_statuses(p, consumer, symbicache)
|
||||||
|
elif type(paths) is dict:
|
||||||
|
for _, p in paths.items():
|
||||||
|
return update_dep_statuses(p, consumer, symbicache)
|
||||||
|
fatal(-1, 'WRONG PATHS TYPE')
|
||||||
|
|
||||||
|
def dep_differ(paths, consumer: str, symbicache: SymbiCache):
|
||||||
|
"""
|
||||||
|
Check if a dependency differs from its last version, lack of dependency is
|
||||||
|
treated as "differs"
|
||||||
|
"""
|
||||||
|
|
||||||
|
if type(paths) is str:
|
||||||
|
s = symbicache.get_status(paths, consumer)
|
||||||
|
if s == 'untracked':
|
||||||
|
symbicache.update(paths, consumer)
|
||||||
|
return symbicache.get_status(paths, consumer) != 'same'
|
||||||
|
elif type(paths) is list:
|
||||||
|
return True in [dep_differ(p, consumer, symbicache) for p in paths]
|
||||||
|
elif type(paths) is dict:
|
||||||
|
return True in [dep_differ(p, consumer, symbicache) \
|
||||||
|
for _, p in paths.items()]
|
||||||
|
return False
|
||||||
|
def dep_will_differ(target: str, paths, consumer: str,
|
||||||
|
os_map: 'dict[str, Stage]', run_stages: 'set[str]',
|
||||||
|
symbicache: SymbiCache):
|
||||||
|
"""
|
||||||
|
Check if a dependency or any of the dependencies it depends on differ from
|
||||||
|
their last versions.
|
||||||
|
"""
|
||||||
|
|
||||||
|
provider = os_map.get(target)
|
||||||
|
if provider:
|
||||||
|
return (provider.name in run_stages) or \
|
||||||
|
dep_differ(paths, consumer, symbicache)
|
||||||
|
return dep_differ(paths, consumer, symbicache)
|
||||||
|
|
||||||
|
def _print_unreachable_stage_message(provider: Stage, take: str):
|
||||||
|
sfprint(0, ' Stage '
|
||||||
|
f'`{Style.BRIGHT + provider.name + Style.RESET_ALL}` is '
|
||||||
|
'unreachable due to unmet dependency '
|
||||||
|
f'`{Style.BRIGHT + take.name + Style.RESET_ALL}`')
|
||||||
|
|
||||||
|
def config_mod_runctx(stage: Stage, platform_name: str, values: 'dict[str, ]',
|
||||||
|
dep_paths: 'dict[str, str | list[str]]',
|
||||||
|
config_paths: 'dict[str, str | list[str]]'):
|
||||||
|
config = prepare_stage_input(stage, platform_name, values,
|
||||||
|
dep_paths, config_paths)
|
||||||
|
return ModRunCtx(share_dir_path, binpath, config)
|
||||||
|
|
||||||
|
class Flow:
|
||||||
|
""" Describes a complete, configured flow, ready for execution. """
|
||||||
|
|
||||||
|
# Dependendecy to build
|
||||||
|
target: str
|
||||||
|
# Values in global scope
|
||||||
|
cfg: FlowConfig
|
||||||
|
# dependency-producer map
|
||||||
|
os_map: 'dict[str, Stage]'
|
||||||
|
# Paths resolved for dependencies
|
||||||
|
dep_paths: 'dict[str, str | list[str]]'
|
||||||
|
# Explicit configs for dependency paths
|
||||||
|
# config_paths: 'dict[str, str | list[str]]'
|
||||||
|
# Stages that need to be run
|
||||||
|
run_stages: 'set[str]'
|
||||||
|
# Number of stages that relied on outdated version of a (checked) dependency
|
||||||
|
deps_rebuilds: 'dict[str, int]'
|
||||||
|
symbicache: 'SymbiCache | None'
|
||||||
|
flow_cfg: FlowConfig
|
||||||
|
|
||||||
|
def __init__(self, target: str, cfg: FlowConfig,
|
||||||
|
symbicache: 'SymbiCache | None'):
|
||||||
|
self.target = target
|
||||||
|
self.os_map = map_outputs_to_stages(cfg.stages.values())
|
||||||
|
|
||||||
|
explicit_deps = cfg.get_dependency_overrides()
|
||||||
|
# print(explicit_deps)
|
||||||
|
|
||||||
|
self.dep_paths = dict(filter_existing_deps(explicit_deps, symbicache))
|
||||||
|
self.run_stages = set()
|
||||||
|
self.symbicache = symbicache
|
||||||
|
self.cfg = cfg
|
||||||
|
self.deps_rebuilds = {}
|
||||||
|
|
||||||
|
self._resolve_dependencies(self.target, set())
|
||||||
|
|
||||||
|
def _dep_will_differ(self, dep: str, paths, consumer: str):
|
||||||
|
if not self.symbicache: # Handle --nocache mode
|
||||||
|
return True
|
||||||
|
return dep_will_differ(dep, paths, consumer,
|
||||||
|
self.os_map, self.run_stages,
|
||||||
|
self.symbicache)
|
||||||
|
|
||||||
|
def _resolve_dependencies(self, dep: str, stages_checked: 'set[str]'):
|
||||||
|
# Initialize the dependency status if necessary
|
||||||
|
if self.deps_rebuilds.get(dep) is None:
|
||||||
|
self.deps_rebuilds[dep] = 0
|
||||||
|
# Check if an explicit dependency is already resolved
|
||||||
|
paths = self.dep_paths.get(dep)
|
||||||
|
if paths and not self.os_map.get(dep):
|
||||||
|
return
|
||||||
|
# Check if a stage can provide the required dependency
|
||||||
|
provider = self.os_map.get(dep)
|
||||||
|
if not provider or provider.name in stages_checked:
|
||||||
|
return
|
||||||
|
|
||||||
|
# TODO: Check if the dependency is "on-demand" and force it in provider's
|
||||||
|
# config if it is.
|
||||||
|
|
||||||
|
for take in provider.takes:
|
||||||
|
self._resolve_dependencies(take.name, stages_checked)
|
||||||
|
# If any of the required dependencies is unavailable, then the
|
||||||
|
# provider stage cannot be run
|
||||||
|
take_paths = self.dep_paths.get(take.name)
|
||||||
|
# Add input path to values (dirty hack)
|
||||||
|
provider.value_overrides[dep_value_str(take.name)] = take_paths
|
||||||
|
|
||||||
|
if not take_paths and take.spec == 'req':
|
||||||
|
_print_unreachable_stage_message(provider, take)
|
||||||
|
return
|
||||||
|
|
||||||
|
if self._dep_will_differ(take.name, take_paths, provider.name):
|
||||||
|
sfprint(2, f'{take.name} is causing rebuild for {provider.name}')
|
||||||
|
self.run_stages.add(provider.name)
|
||||||
|
self.deps_rebuilds[take.name] += 1
|
||||||
|
|
||||||
|
stage_values = self.cfg.get_r_env(provider.name).values
|
||||||
|
modrunctx = config_mod_runctx(provider, self.cfg.platform,
|
||||||
|
stage_values, self.dep_paths,
|
||||||
|
self.cfg.get_dependency_overrides())
|
||||||
|
|
||||||
|
outputs = module_map(provider.module, modrunctx)
|
||||||
|
|
||||||
|
stages_checked.add(provider.name)
|
||||||
|
self.dep_paths.update(outputs)
|
||||||
|
|
||||||
|
for _, out_paths in outputs.items():
|
||||||
|
if (out_paths is not None) and not (req_exists(out_paths)):
|
||||||
|
self.run_stages.add(provider.name)
|
||||||
|
|
||||||
|
# Verify module's outputs and add paths as values.
|
||||||
|
outs = outputs.keys()
|
||||||
|
# print(outs)
|
||||||
|
for o in provider.produces:
|
||||||
|
if o.name not in outs:
|
||||||
|
if o.spec == 'req' or (o.spec == 'demand' and \
|
||||||
|
o.name in self.cfg.get_dependency_overrides().keys()):
|
||||||
|
fatal(-1, f'Module {provider.name} did not produce a mapping '
|
||||||
|
f'for a required output `{o.name}`')
|
||||||
|
else:
|
||||||
|
# Remove an on-demand/optional output that is not produced
|
||||||
|
# from os_map.
|
||||||
|
self.os_map.pop(o.name)
|
||||||
|
# Add a value for the output (dirty ack yet again)
|
||||||
|
o_path = outputs.get(o.name)
|
||||||
|
|
||||||
|
if o_path is not None:
|
||||||
|
provider.value_overrides[dep_value_str(o.name)] = \
|
||||||
|
outputs.get(o.name)
|
||||||
|
|
||||||
|
|
||||||
|
def print_resolved_dependencies(self, verbosity: int):
|
||||||
|
deps = list(self.deps_rebuilds.keys())
|
||||||
|
deps.sort()
|
||||||
|
|
||||||
|
for dep in deps:
|
||||||
|
status = Fore.RED + '[X]' + Fore.RESET
|
||||||
|
source = Fore.YELLOW + 'MISSING' + Fore.RESET
|
||||||
|
paths = self.dep_paths.get(dep)
|
||||||
|
|
||||||
|
if paths:
|
||||||
|
exists = req_exists(paths)
|
||||||
|
provider = self.os_map.get(dep)
|
||||||
|
if provider and provider.name in self.run_stages:
|
||||||
|
if exists:
|
||||||
|
status = Fore.YELLOW + '[R]' + Fore.RESET
|
||||||
|
else:
|
||||||
|
status = Fore.YELLOW + '[S]' + Fore.RESET
|
||||||
|
source = f'{Fore.BLUE + self.os_map[dep].name + Fore.RESET} ' \
|
||||||
|
f'-> {paths}'
|
||||||
|
elif exists:
|
||||||
|
if self.deps_rebuilds[dep] > 0:
|
||||||
|
status = Fore.GREEN + '[N]' + Fore.RESET
|
||||||
|
else:
|
||||||
|
status = Fore.GREEN + '[O]' + Fore.RESET
|
||||||
|
source = paths
|
||||||
|
elif self.os_map.get(dep):
|
||||||
|
status = Fore.RED + '[U]' + Fore.RESET
|
||||||
|
source = \
|
||||||
|
f'{Fore.BLUE + self.os_map[dep].name + Fore.RESET} -> ???'
|
||||||
|
|
||||||
|
sfprint(verbosity, f' {Style.BRIGHT + status} '
|
||||||
|
f'{dep + Style.RESET_ALL}: {source}')
|
||||||
|
|
||||||
|
def _build_dep(self, dep):
|
||||||
|
paths = self.dep_paths.get(dep)
|
||||||
|
provider = self.os_map.get(dep)
|
||||||
|
run = (provider.name in self.run_stages) if provider else False
|
||||||
|
if not paths:
|
||||||
|
sfprint(2, f'Dependency {dep} is unresolved.')
|
||||||
|
return False
|
||||||
|
|
||||||
|
if req_exists(paths) and not run:
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
assert(provider)
|
||||||
|
|
||||||
|
any_dep_differ = False if self.symbicache else True
|
||||||
|
for p_dep in provider.takes:
|
||||||
|
if not self._build_dep(p_dep.name):
|
||||||
|
assert (p_dep.spec != 'req')
|
||||||
|
continue
|
||||||
|
|
||||||
|
if self.symbicache:
|
||||||
|
any_dep_differ |= \
|
||||||
|
update_dep_statuses(self.dep_paths[p_dep.name],
|
||||||
|
provider.name, self.symbicache)
|
||||||
|
|
||||||
|
# If dependencies remained the same, consider the dep as up-to date
|
||||||
|
# For example, when changing a comment in Verilog source code,
|
||||||
|
# the initial dependency resolution will report a need for complete
|
||||||
|
# rebuild, however, after the synthesis stage, the generated eblif
|
||||||
|
# will reamin the same, thus making it unnecessary to continue the
|
||||||
|
# rebuild process.
|
||||||
|
if (not any_dep_differ) and req_exists(paths):
|
||||||
|
sfprint(2, f'Skipping rebuild of `'
|
||||||
|
f'{Style.BRIGHT + dep + Style.RESET_ALL}` because all '
|
||||||
|
f'of it\'s dependencies remained unchanged')
|
||||||
|
return True
|
||||||
|
|
||||||
|
stage_values = self.cfg.get_r_env(provider.name).values
|
||||||
|
modrunctx = config_mod_runctx(provider, self.cfg.platform,
|
||||||
|
stage_values, self.dep_paths,
|
||||||
|
self.cfg.get_dependency_overrides())
|
||||||
|
module_exec(provider.module, modrunctx)
|
||||||
|
|
||||||
|
self.run_stages.discard(provider.name)
|
||||||
|
|
||||||
|
if not req_exists(paths):
|
||||||
|
raise DependencyNotProducedException(dep, provider.name)
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
def execute(self):
|
||||||
|
self._build_dep(self.target)
|
||||||
|
if self.symbicache:
|
||||||
|
update_dep_statuses(self.dep_paths[self.target], '__target',
|
||||||
|
self.symbicache)
|
||||||
|
sfprint(0, f'Target `{Style.BRIGHT + self.target + Style.RESET_ALL}` '
|
||||||
|
f'-> {self.dep_paths[self.target]}')
|
||||||
|
|
||||||
|
def display_dep_info(stages: 'Iterable[Stage]'):
|
||||||
|
sfprint(0, 'Platform dependencies/targets:')
|
||||||
|
longest_out_name_len = 0
|
||||||
|
for stage in stages:
|
||||||
|
for out in stage.produces:
|
||||||
|
l = len(out.name)
|
||||||
|
if l > longest_out_name_len:
|
||||||
|
longest_out_name_len = l
|
||||||
|
|
||||||
|
desc_indent = longest_out_name_len + 7
|
||||||
|
nl_indentstr = '\n'
|
||||||
|
for _ in range(0, desc_indent):
|
||||||
|
nl_indentstr += ' '
|
||||||
|
|
||||||
|
for stage in stages:
|
||||||
|
for out in stage.produces:
|
||||||
|
pname = Style.BRIGHT + out.name + Style.RESET_ALL
|
||||||
|
indent = ''
|
||||||
|
for _ in range(0, desc_indent - len(pname) + 3):
|
||||||
|
indent += ' '
|
||||||
|
specstr = '???'
|
||||||
|
if out.spec == 'req':
|
||||||
|
specstr = f'{Fore.BLUE}guaranteed{Fore.RESET}'
|
||||||
|
elif out.spec == 'maybe':
|
||||||
|
specstr = f'{Fore.YELLOW}not guaranteed{Fore.RESET}'
|
||||||
|
elif out.spec == 'demand':
|
||||||
|
specstr = f'{Fore.RED}on-demand{Fore.RESET}'
|
||||||
|
pgen = f'{Style.DIM}stage: `{stage.name}`, '\
|
||||||
|
f'spec: {specstr}{Style.RESET_ALL}'
|
||||||
|
pdesc = stage.meta[out.name].replace('\n', nl_indentstr)
|
||||||
|
sfprint(0, f' {Style.BRIGHT + out.name + Style.RESET_ALL}:'
|
||||||
|
f'{indent}{pdesc}{nl_indentstr}{pgen}')
|
||||||
|
|
||||||
|
def display_stage_info(stage: Stage):
|
||||||
|
if stage is None:
|
||||||
|
sfprint(0, f'Stage does not exist')
|
||||||
|
sfbuild_fail()
|
||||||
|
return
|
||||||
|
|
||||||
|
sfprint(0, f'Stage `{Style.BRIGHT}{stage.name}{Style.RESET_ALL}`:')
|
||||||
|
sfprint(0, f' Module: `{Style.BRIGHT}{stage.module.name}{Style.RESET_ALL}`')
|
||||||
|
sfprint(0, f' Module info:')
|
||||||
|
|
||||||
|
mod_info = get_module_info(stage.module)
|
||||||
|
mod_info = '\n '.join(mod_info.split('\n'))
|
||||||
|
|
||||||
|
sfprint(0, f' {mod_info}')
|
||||||
|
|
||||||
|
sfbuild_done_str = Style.BRIGHT + Fore.GREEN + 'DONE'
|
||||||
|
sfbuild_silent = 0
|
||||||
|
|
||||||
|
def sfbuild_fail():
|
||||||
|
global sfbuild_done_str
|
||||||
|
sfbuild_done_str = Style.BRIGHT + Fore.RED + 'FAILED'
|
||||||
|
|
||||||
|
def sfbuild_done():
|
||||||
|
sfprint(1, f'sfbuild: {sfbuild_done_str}'
|
||||||
|
f'{Style.RESET_ALL + Fore.RESET}')
|
||||||
|
exit(0)
|
||||||
|
|
||||||
|
def setup_resolution_env():
|
||||||
|
""" Sets up a ResolutionEnv with sfbuild's default built-ins. """
|
||||||
|
|
||||||
|
r_env = ResolutionEnv({
|
||||||
|
'shareDir': share_dir_path,
|
||||||
|
'binDir': os.path.realpath(os.path.join(share_dir_path, '../../bin'))
|
||||||
|
})
|
||||||
|
r_env.add_values(sf_ugly.generate_values())
|
||||||
|
return r_env
|
||||||
|
|
||||||
|
def open_project_flow_config(path: str) -> ProjectFlowConfig:
|
||||||
|
try:
|
||||||
|
flow_cfg = open_project_flow_cfg(path)
|
||||||
|
except FileNotFoundError as _:
|
||||||
|
fatal(-1, 'The provided flow configuration file does not exist')
|
||||||
|
return flow_cfg
|
||||||
|
|
||||||
|
def verify_platform_stage_params(flow_cfg: FlowConfig,
|
||||||
|
platform: 'str | None' = None,
|
||||||
|
stage: 'str | None' = None):
|
||||||
|
if platform:
|
||||||
|
if not verify_platform_name(platform, mypath):
|
||||||
|
sfprint(0, f'Platform `{platform}`` is unsupported.')
|
||||||
|
return False
|
||||||
|
if args.platform not in flow_cfg.platforms():
|
||||||
|
sfprint(0, f'Platform `{platform}`` is not in project.')
|
||||||
|
return False
|
||||||
|
|
||||||
|
if stage:
|
||||||
|
if not verify_stage(platform, stage, mypath):
|
||||||
|
sfprint(0, f'Stage `{stage}` is invalid.')
|
||||||
|
sfbuild_fail()
|
||||||
|
return False
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
def get_platform_name_for_part(part_name: str):
|
||||||
|
"""
|
||||||
|
Gets a name that identifies the platform setup required for a specific chip.
|
||||||
|
The reason for such distinction is that plenty of chips with different names
|
||||||
|
differ only in a type of package they use.
|
||||||
|
"""
|
||||||
|
|
||||||
|
d: dict
|
||||||
|
with open(os.path.join(mypath, 'part_db/parts.json')) as f:
|
||||||
|
d = json.loads(f.read())
|
||||||
|
return d.get(part_name.upper())
|
||||||
|
|
||||||
|
def cmd_build(args: Namespace):
|
||||||
|
""" sfbuild's `build` command implementation """
|
||||||
|
|
||||||
|
project_flow_cfg: ProjectFlowConfig = None
|
||||||
|
|
||||||
|
|
||||||
|
platform = args.platform
|
||||||
|
if platform is None:
|
||||||
|
if args.part:
|
||||||
|
platform = get_platform_name_for_part(args.part)
|
||||||
|
|
||||||
|
if args.flow:
|
||||||
|
project_flow_cfg = open_project_flow_config(args.flow)
|
||||||
|
elif platform is not None:
|
||||||
|
project_flow_cfg = ProjectFlowConfig('.temp.flow.json')
|
||||||
|
project_flow_cfg.flow_cfg = get_cli_flow_config(args, platform)
|
||||||
|
if platform is None and project_flow_cfg is not None:
|
||||||
|
platform = project_flow_cfg.get_default_platform()
|
||||||
|
if platform is None:
|
||||||
|
fatal(-1, 'You have to specify a platform name or a part name or '
|
||||||
|
'configure a default platform.')
|
||||||
|
if platform is None or project_flow_cfg is None:
|
||||||
|
fatal(-1, 'No configuration was provided. Use `--flow`, `--platform` or '
|
||||||
|
'`--part` to configure flow..')
|
||||||
|
|
||||||
|
platform_path = os.path.join(mypath, 'platforms', platform + '.json')
|
||||||
|
platform_def = None
|
||||||
|
try:
|
||||||
|
with open(platform_path) as platform_file:
|
||||||
|
platform_def = platform_file.read()
|
||||||
|
except FileNotFoundError as _:
|
||||||
|
fatal(-1, f'The platform flow definition file {platform_path} for the platform '
|
||||||
|
f'{platform} referenced in flow definition file {args.flow} '
|
||||||
|
'cannot be found.')
|
||||||
|
|
||||||
|
r_env = setup_resolution_env()
|
||||||
|
|
||||||
|
sfprint(2, 'Scanning modules...')
|
||||||
|
scan_modules(mypath)
|
||||||
|
|
||||||
|
flow_definition_dict = json.loads(platform_def)
|
||||||
|
flow_def = FlowDefinition(flow_definition_dict, r_env)
|
||||||
|
flow_cfg = FlowConfig(project_flow_cfg, flow_def, platform)
|
||||||
|
|
||||||
|
|
||||||
|
if len(flow_cfg.stages) == 0:
|
||||||
|
fatal(-1, 'Platform flow does not define any stage')
|
||||||
|
|
||||||
|
if args.info:
|
||||||
|
display_dep_info(flow_cfg.stages.values())
|
||||||
|
sfbuild_done()
|
||||||
|
|
||||||
|
if args.stageinfo:
|
||||||
|
display_stage_info(flow_cfg.stages.get(args.stageinfo[0]))
|
||||||
|
sfbuild_done()
|
||||||
|
|
||||||
|
target = args.target
|
||||||
|
if target is None:
|
||||||
|
target = project_flow_cfg.get_default_target(platform)
|
||||||
|
if target is None:
|
||||||
|
fatal(-1, 'Please specify desired target using `--target` option '
|
||||||
|
'or configure a default target.')
|
||||||
|
|
||||||
|
flow = Flow(
|
||||||
|
target=target,
|
||||||
|
cfg=flow_cfg,
|
||||||
|
symbicache=SymbiCache(SYMBICACHEPATH) if not args.nocache else None
|
||||||
|
)
|
||||||
|
|
||||||
|
dep_print_verbosity = 0 if args.pretend else 2
|
||||||
|
sfprint(dep_print_verbosity, '\nProject status:')
|
||||||
|
flow.print_resolved_dependencies(dep_print_verbosity)
|
||||||
|
sfprint(dep_print_verbosity, '')
|
||||||
|
|
||||||
|
if args.pretend:
|
||||||
|
sfbuild_done()
|
||||||
|
|
||||||
|
try:
|
||||||
|
flow.execute()
|
||||||
|
except Exception as e:
|
||||||
|
sfprint(0, e)
|
||||||
|
sfbuild_fail()
|
||||||
|
|
||||||
|
if flow.symbicache:
|
||||||
|
flow.symbicache.save()
|
||||||
|
|
||||||
|
def cmd_show_dependencies(args: Namespace):
|
||||||
|
""" sfbuild's `showd` command implementation """
|
||||||
|
|
||||||
|
flow_cfg = open_project_flow_config(args.flow)
|
||||||
|
|
||||||
|
if not verify_platform_stage_params(flow_cfg, args.platform):
|
||||||
|
sfbuild_fail()
|
||||||
|
return
|
||||||
|
|
||||||
|
platform_overrides: 'set | None' = None
|
||||||
|
if args.platform is not None:
|
||||||
|
platform_overrides = \
|
||||||
|
set(flow_cfg.get_dependency_platform_overrides(args.platform).keys())
|
||||||
|
|
||||||
|
display_list = []
|
||||||
|
|
||||||
|
raw_deps = flow_cfg.get_dependencies_raw(args.platform)
|
||||||
|
|
||||||
|
for dep_name, dep_paths in raw_deps.items():
|
||||||
|
prstr: str
|
||||||
|
if (platform_overrides is not None) and (dep_name in platform_overrides):
|
||||||
|
prstr = f'{Style.DIM}({args.platform}){Style.RESET_ALL} ' \
|
||||||
|
f'{Style.BRIGHT + dep_name + Style.RESET_ALL}: {dep_paths}'
|
||||||
|
else:
|
||||||
|
prstr = f'{Style.BRIGHT + dep_name + Style.RESET_ALL}: {dep_paths}'
|
||||||
|
|
||||||
|
display_list.append((dep_name, prstr))
|
||||||
|
|
||||||
|
display_list.sort(key = lambda p: p[0])
|
||||||
|
|
||||||
|
for _, prstr in display_list:
|
||||||
|
sfprint(0, prstr)
|
||||||
|
|
||||||
|
set_verbosity_level(-1)
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = setup_argparser()
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
set_verbosity_level(args.verbose - (1 if args.silent else 0))
|
||||||
|
|
||||||
|
if args.command == 'build':
|
||||||
|
cmd_build(args)
|
||||||
|
sfbuild_done()
|
||||||
|
|
||||||
|
if args.command == 'showd':
|
||||||
|
cmd_show_dependencies(args)
|
||||||
|
sfbuild_done()
|
||||||
|
|
||||||
|
sfprint(0, 'Please use a command.\nUse `--help` flag to learn more.')
|
||||||
|
sfbuild_done()
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
|
|
@ -91,7 +91,7 @@ setuptools_setup(
|
||||||
install_requires=list(set(get_requirements(requirementsFile))),
|
install_requires=list(set(get_requirements(requirementsFile))),
|
||||||
entry_points={
|
entry_points={
|
||||||
"console_scripts": [
|
"console_scripts": [
|
||||||
"f4pga = f4pga.sfbuild:main",
|
"f4pga = f4pga.__init__:main",
|
||||||
] + wrapper_entrypoints
|
] + wrapper_entrypoints
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
|
||||||
645
f4pga/sfbuild.py
645
f4pga/sfbuild.py
|
|
@ -1,645 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
|
|
||||||
"""
|
|
||||||
sfbuild - Symbiflow Build System
|
|
||||||
|
|
||||||
This tool allows for building FPGA targets (such as bitstreams) for any supported
|
|
||||||
platform with just one simple command and a project file.
|
|
||||||
|
|
||||||
The idea is that sfbuild wraps all the tools needed by different platforms in
|
|
||||||
"modules", which define inputs/outputs and various parameters. This allows
|
|
||||||
sfbuild to resolve dependencies for any target provided that a "flow definition"
|
|
||||||
file exists for such target. The flow defeinition file list modules available for
|
|
||||||
that platform and may tweak some settings of those modules.
|
|
||||||
|
|
||||||
A basic example of using sfbuild:
|
|
||||||
$ sfbuild build --platform arty_35 -t bitstream
|
|
||||||
|
|
||||||
This will make sfbuild attempt to create a bitstream for arty_35 platform.
|
|
||||||
flow.json is a flow configuration file, which should be created for a project
|
|
||||||
that uses sfbuild. Iontains project-specific definitions needed within the flow,
|
|
||||||
such as list of source code files.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from pathlib import Path
|
|
||||||
from argparse import Namespace
|
|
||||||
import os
|
|
||||||
from os import environ
|
|
||||||
import json
|
|
||||||
from typing import Iterable
|
|
||||||
from colorama import Fore, Style
|
|
||||||
from f4pga.sf_common import ResolutionEnv, fatal, scan_modules, set_verbosity_level, \
|
|
||||||
sfprint
|
|
||||||
from f4pga.sf_module import *
|
|
||||||
from f4pga.sf_cache import SymbiCache
|
|
||||||
import f4pga.sf_ugly as sf_ugly
|
|
||||||
from f4pga.sf_flow_config import ProjectFlowConfig, FlowConfig, FlowDefinition, \
|
|
||||||
open_project_flow_cfg, verify_platform_name, \
|
|
||||||
verify_stage
|
|
||||||
from f4pga.sf_module_runner import *
|
|
||||||
from f4pga.sf_module_inspector import get_module_info
|
|
||||||
from f4pga.sf_stage import Stage
|
|
||||||
from f4pga.sf_argparse import setup_argparser, get_cli_flow_config
|
|
||||||
|
|
||||||
SYMBICACHEPATH = '.symbicache'
|
|
||||||
|
|
||||||
binpath = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(os.sys.argv[0])), '..'))
|
|
||||||
mypath = str(Path(__file__).resolve().parent)
|
|
||||||
|
|
||||||
share_dir_path = os.path.realpath(f"{environ.get('INSTALL_DIR', '/usr/local')}/xc7/install/share/symbiflow")
|
|
||||||
|
|
||||||
class DependencyNotProducedException(Exception):
|
|
||||||
dep_name: str
|
|
||||||
provider: str
|
|
||||||
|
|
||||||
def __init__(self, dep_name: str, provider: str):
|
|
||||||
self.dep_name = dep_name
|
|
||||||
self.provider = provider
|
|
||||||
|
|
||||||
def __str__(self) -> str:
|
|
||||||
return f'Stage `{self.provider}` did not produce promised ' \
|
|
||||||
f'dependency `{self.dep_name}`'
|
|
||||||
|
|
||||||
def dep_value_str(dep: str):
|
|
||||||
return ':' + dep
|
|
||||||
|
|
||||||
def platform_stages(platform_flow, r_env):
|
|
||||||
""" Iterates over all stages available in a given flow. """
|
|
||||||
|
|
||||||
stage_options = platform_flow.get('stage_options')
|
|
||||||
for stage_name, modulestr in platform_flow['stages'].items():
|
|
||||||
mod_opts = stage_options.get(stage_name) if stage_options else None
|
|
||||||
yield Stage(stage_name, modulestr, mod_opts, r_env)
|
|
||||||
|
|
||||||
def req_exists(r):
|
|
||||||
""" Checks whether a dependency exists on a drive. """
|
|
||||||
|
|
||||||
if type(r) is str:
|
|
||||||
if not os.path.isfile(r) and not os.path.islink(r) \
|
|
||||||
and not os.path.isdir(r):
|
|
||||||
return False
|
|
||||||
elif type(r) is list:
|
|
||||||
return not (False in map(req_exists, r))
|
|
||||||
else:
|
|
||||||
raise Exception(f'Requirements can be currently checked only for single '
|
|
||||||
f'paths, or path lists (reason: {r})')
|
|
||||||
return True
|
|
||||||
|
|
||||||
def map_outputs_to_stages(stages: 'list[Stage]'):
|
|
||||||
"""
|
|
||||||
Associates a stage with every possible output.
|
|
||||||
This is commonly refferef to as `os_map` (output-stage-map) through the code.
|
|
||||||
"""
|
|
||||||
|
|
||||||
os_map: 'dict[str, Stage]' = {} # Output-Stage map
|
|
||||||
for stage in stages:
|
|
||||||
for output in stage.produces:
|
|
||||||
if not os_map.get(output.name):
|
|
||||||
os_map[output.name] = stage
|
|
||||||
elif os_map[output.name] != stage:
|
|
||||||
raise Exception(f'Dependency `{output.name}` is generated by '
|
|
||||||
f'stage `{os_map[output.name].name}` and '
|
|
||||||
f'`{stage.name}`. Dependencies can have only one '
|
|
||||||
'provider at most.')
|
|
||||||
return os_map
|
|
||||||
|
|
||||||
def filter_existing_deps(deps: 'dict[str, ]', symbicache):
|
|
||||||
return [(n, p) for n, p in deps.items() \
|
|
||||||
if req_exists(p)] # and not dep_differ(p, symbicache)]
|
|
||||||
|
|
||||||
def get_stage_values_override(og_values: dict, stage: Stage):
|
|
||||||
values = og_values.copy()
|
|
||||||
values.update(stage.value_ovds)
|
|
||||||
return values
|
|
||||||
|
|
||||||
def prepare_stage_io_input(stage: Stage):
|
|
||||||
return { 'params': stage.params } if stage.params is not None else {}
|
|
||||||
|
|
||||||
def prepare_stage_input(stage: Stage, platform_name: str, values: dict,
|
|
||||||
dep_paths: 'dict[str, ]', config_paths: 'dict[str, ]'):
|
|
||||||
takes = {}
|
|
||||||
for take in stage.takes:
|
|
||||||
paths = dep_paths.get(take.name)
|
|
||||||
if paths: # Some takes may be not required
|
|
||||||
takes[take.name] = paths
|
|
||||||
|
|
||||||
produces = {}
|
|
||||||
for prod in stage.produces:
|
|
||||||
if dep_paths.get(prod.name):
|
|
||||||
produces[prod.name] = dep_paths[prod.name]
|
|
||||||
elif config_paths.get(prod.name):
|
|
||||||
produces[prod.name] = config_paths[prod.name]
|
|
||||||
|
|
||||||
stage_mod_cfg = {
|
|
||||||
'takes': takes,
|
|
||||||
'produces': produces,
|
|
||||||
'values': values,
|
|
||||||
'platform': platform_name,
|
|
||||||
}
|
|
||||||
|
|
||||||
return stage_mod_cfg
|
|
||||||
|
|
||||||
def update_dep_statuses(paths, consumer: str, symbicache: SymbiCache):
|
|
||||||
if type(paths) is str:
|
|
||||||
return symbicache.update(paths, consumer)
|
|
||||||
elif type(paths) is list:
|
|
||||||
for p in paths:
|
|
||||||
return update_dep_statuses(p, consumer, symbicache)
|
|
||||||
elif type(paths) is dict:
|
|
||||||
for _, p in paths.items():
|
|
||||||
return update_dep_statuses(p, consumer, symbicache)
|
|
||||||
fatal(-1, 'WRONG PATHS TYPE')
|
|
||||||
|
|
||||||
def dep_differ(paths, consumer: str, symbicache: SymbiCache):
|
|
||||||
"""
|
|
||||||
Check if a dependency differs from its last version, lack of dependency is
|
|
||||||
treated as "differs"
|
|
||||||
"""
|
|
||||||
|
|
||||||
if type(paths) is str:
|
|
||||||
s = symbicache.get_status(paths, consumer)
|
|
||||||
if s == 'untracked':
|
|
||||||
symbicache.update(paths, consumer)
|
|
||||||
return symbicache.get_status(paths, consumer) != 'same'
|
|
||||||
elif type(paths) is list:
|
|
||||||
return True in [dep_differ(p, consumer, symbicache) for p in paths]
|
|
||||||
elif type(paths) is dict:
|
|
||||||
return True in [dep_differ(p, consumer, symbicache) \
|
|
||||||
for _, p in paths.items()]
|
|
||||||
return False
|
|
||||||
def dep_will_differ(target: str, paths, consumer: str,
|
|
||||||
os_map: 'dict[str, Stage]', run_stages: 'set[str]',
|
|
||||||
symbicache: SymbiCache):
|
|
||||||
"""
|
|
||||||
Check if a dependency or any of the dependencies it depends on differ from
|
|
||||||
their last versions.
|
|
||||||
"""
|
|
||||||
|
|
||||||
provider = os_map.get(target)
|
|
||||||
if provider:
|
|
||||||
return (provider.name in run_stages) or \
|
|
||||||
dep_differ(paths, consumer, symbicache)
|
|
||||||
return dep_differ(paths, consumer, symbicache)
|
|
||||||
|
|
||||||
def _print_unreachable_stage_message(provider: Stage, take: str):
|
|
||||||
sfprint(0, ' Stage '
|
|
||||||
f'`{Style.BRIGHT + provider.name + Style.RESET_ALL}` is '
|
|
||||||
'unreachable due to unmet dependency '
|
|
||||||
f'`{Style.BRIGHT + take.name + Style.RESET_ALL}`')
|
|
||||||
|
|
||||||
def config_mod_runctx(stage: Stage, platform_name: str, values: 'dict[str, ]',
|
|
||||||
dep_paths: 'dict[str, str | list[str]]',
|
|
||||||
config_paths: 'dict[str, str | list[str]]'):
|
|
||||||
config = prepare_stage_input(stage, platform_name, values,
|
|
||||||
dep_paths, config_paths)
|
|
||||||
return ModRunCtx(share_dir_path, binpath, config)
|
|
||||||
|
|
||||||
class Flow:
|
|
||||||
""" Describes a complete, configured flow, ready for execution. """
|
|
||||||
|
|
||||||
# Dependendecy to build
|
|
||||||
target: str
|
|
||||||
# Values in global scope
|
|
||||||
cfg: FlowConfig
|
|
||||||
# dependency-producer map
|
|
||||||
os_map: 'dict[str, Stage]'
|
|
||||||
# Paths resolved for dependencies
|
|
||||||
dep_paths: 'dict[str, str | list[str]]'
|
|
||||||
# Explicit configs for dependency paths
|
|
||||||
# config_paths: 'dict[str, str | list[str]]'
|
|
||||||
# Stages that need to be run
|
|
||||||
run_stages: 'set[str]'
|
|
||||||
# Number of stages that relied on outdated version of a (checked) dependency
|
|
||||||
deps_rebuilds: 'dict[str, int]'
|
|
||||||
symbicache: 'SymbiCache | None'
|
|
||||||
flow_cfg: FlowConfig
|
|
||||||
|
|
||||||
def __init__(self, target: str, cfg: FlowConfig,
|
|
||||||
symbicache: 'SymbiCache | None'):
|
|
||||||
self.target = target
|
|
||||||
self.os_map = map_outputs_to_stages(cfg.stages.values())
|
|
||||||
|
|
||||||
explicit_deps = cfg.get_dependency_overrides()
|
|
||||||
# print(explicit_deps)
|
|
||||||
|
|
||||||
self.dep_paths = dict(filter_existing_deps(explicit_deps, symbicache))
|
|
||||||
self.run_stages = set()
|
|
||||||
self.symbicache = symbicache
|
|
||||||
self.cfg = cfg
|
|
||||||
self.deps_rebuilds = {}
|
|
||||||
|
|
||||||
self._resolve_dependencies(self.target, set())
|
|
||||||
|
|
||||||
def _dep_will_differ(self, dep: str, paths, consumer: str):
|
|
||||||
if not self.symbicache: # Handle --nocache mode
|
|
||||||
return True
|
|
||||||
return dep_will_differ(dep, paths, consumer,
|
|
||||||
self.os_map, self.run_stages,
|
|
||||||
self.symbicache)
|
|
||||||
|
|
||||||
def _resolve_dependencies(self, dep: str, stages_checked: 'set[str]'):
|
|
||||||
# Initialize the dependency status if necessary
|
|
||||||
if self.deps_rebuilds.get(dep) is None:
|
|
||||||
self.deps_rebuilds[dep] = 0
|
|
||||||
# Check if an explicit dependency is already resolved
|
|
||||||
paths = self.dep_paths.get(dep)
|
|
||||||
if paths and not self.os_map.get(dep):
|
|
||||||
return
|
|
||||||
# Check if a stage can provide the required dependency
|
|
||||||
provider = self.os_map.get(dep)
|
|
||||||
if not provider or provider.name in stages_checked:
|
|
||||||
return
|
|
||||||
|
|
||||||
# TODO: Check if the dependency is "on-demand" and force it in provider's
|
|
||||||
# config if it is.
|
|
||||||
|
|
||||||
for take in provider.takes:
|
|
||||||
self._resolve_dependencies(take.name, stages_checked)
|
|
||||||
# If any of the required dependencies is unavailable, then the
|
|
||||||
# provider stage cannot be run
|
|
||||||
take_paths = self.dep_paths.get(take.name)
|
|
||||||
# Add input path to values (dirty hack)
|
|
||||||
provider.value_overrides[dep_value_str(take.name)] = take_paths
|
|
||||||
|
|
||||||
if not take_paths and take.spec == 'req':
|
|
||||||
_print_unreachable_stage_message(provider, take)
|
|
||||||
return
|
|
||||||
|
|
||||||
if self._dep_will_differ(take.name, take_paths, provider.name):
|
|
||||||
sfprint(2, f'{take.name} is causing rebuild for {provider.name}')
|
|
||||||
self.run_stages.add(provider.name)
|
|
||||||
self.deps_rebuilds[take.name] += 1
|
|
||||||
|
|
||||||
stage_values = self.cfg.get_r_env(provider.name).values
|
|
||||||
modrunctx = config_mod_runctx(provider, self.cfg.platform,
|
|
||||||
stage_values, self.dep_paths,
|
|
||||||
self.cfg.get_dependency_overrides())
|
|
||||||
|
|
||||||
outputs = module_map(provider.module, modrunctx)
|
|
||||||
|
|
||||||
stages_checked.add(provider.name)
|
|
||||||
self.dep_paths.update(outputs)
|
|
||||||
|
|
||||||
for _, out_paths in outputs.items():
|
|
||||||
if (out_paths is not None) and not (req_exists(out_paths)):
|
|
||||||
self.run_stages.add(provider.name)
|
|
||||||
|
|
||||||
# Verify module's outputs and add paths as values.
|
|
||||||
outs = outputs.keys()
|
|
||||||
# print(outs)
|
|
||||||
for o in provider.produces:
|
|
||||||
if o.name not in outs:
|
|
||||||
if o.spec == 'req' or (o.spec == 'demand' and \
|
|
||||||
o.name in self.cfg.get_dependency_overrides().keys()):
|
|
||||||
fatal(-1, f'Module {provider.name} did not produce a mapping '
|
|
||||||
f'for a required output `{o.name}`')
|
|
||||||
else:
|
|
||||||
# Remove an on-demand/optional output that is not produced
|
|
||||||
# from os_map.
|
|
||||||
self.os_map.pop(o.name)
|
|
||||||
# Add a value for the output (dirty ack yet again)
|
|
||||||
o_path = outputs.get(o.name)
|
|
||||||
|
|
||||||
if o_path is not None:
|
|
||||||
provider.value_overrides[dep_value_str(o.name)] = \
|
|
||||||
outputs.get(o.name)
|
|
||||||
|
|
||||||
|
|
||||||
def print_resolved_dependencies(self, verbosity: int):
|
|
||||||
deps = list(self.deps_rebuilds.keys())
|
|
||||||
deps.sort()
|
|
||||||
|
|
||||||
for dep in deps:
|
|
||||||
status = Fore.RED + '[X]' + Fore.RESET
|
|
||||||
source = Fore.YELLOW + 'MISSING' + Fore.RESET
|
|
||||||
paths = self.dep_paths.get(dep)
|
|
||||||
|
|
||||||
if paths:
|
|
||||||
exists = req_exists(paths)
|
|
||||||
provider = self.os_map.get(dep)
|
|
||||||
if provider and provider.name in self.run_stages:
|
|
||||||
if exists:
|
|
||||||
status = Fore.YELLOW + '[R]' + Fore.RESET
|
|
||||||
else:
|
|
||||||
status = Fore.YELLOW + '[S]' + Fore.RESET
|
|
||||||
source = f'{Fore.BLUE + self.os_map[dep].name + Fore.RESET} ' \
|
|
||||||
f'-> {paths}'
|
|
||||||
elif exists:
|
|
||||||
if self.deps_rebuilds[dep] > 0:
|
|
||||||
status = Fore.GREEN + '[N]' + Fore.RESET
|
|
||||||
else:
|
|
||||||
status = Fore.GREEN + '[O]' + Fore.RESET
|
|
||||||
source = paths
|
|
||||||
elif self.os_map.get(dep):
|
|
||||||
status = Fore.RED + '[U]' + Fore.RESET
|
|
||||||
source = \
|
|
||||||
f'{Fore.BLUE + self.os_map[dep].name + Fore.RESET} -> ???'
|
|
||||||
|
|
||||||
sfprint(verbosity, f' {Style.BRIGHT + status} '
|
|
||||||
f'{dep + Style.RESET_ALL}: {source}')
|
|
||||||
|
|
||||||
def _build_dep(self, dep):
|
|
||||||
paths = self.dep_paths.get(dep)
|
|
||||||
provider = self.os_map.get(dep)
|
|
||||||
run = (provider.name in self.run_stages) if provider else False
|
|
||||||
if not paths:
|
|
||||||
sfprint(2, f'Dependency {dep} is unresolved.')
|
|
||||||
return False
|
|
||||||
|
|
||||||
if req_exists(paths) and not run:
|
|
||||||
return True
|
|
||||||
else:
|
|
||||||
assert(provider)
|
|
||||||
|
|
||||||
any_dep_differ = False if self.symbicache else True
|
|
||||||
for p_dep in provider.takes:
|
|
||||||
if not self._build_dep(p_dep.name):
|
|
||||||
assert (p_dep.spec != 'req')
|
|
||||||
continue
|
|
||||||
|
|
||||||
if self.symbicache:
|
|
||||||
any_dep_differ |= \
|
|
||||||
update_dep_statuses(self.dep_paths[p_dep.name],
|
|
||||||
provider.name, self.symbicache)
|
|
||||||
|
|
||||||
# If dependencies remained the same, consider the dep as up-to date
|
|
||||||
# For example, when changing a comment in Verilog source code,
|
|
||||||
# the initial dependency resolution will report a need for complete
|
|
||||||
# rebuild, however, after the synthesis stage, the generated eblif
|
|
||||||
# will reamin the same, thus making it unnecessary to continue the
|
|
||||||
# rebuild process.
|
|
||||||
if (not any_dep_differ) and req_exists(paths):
|
|
||||||
sfprint(2, f'Skipping rebuild of `'
|
|
||||||
f'{Style.BRIGHT + dep + Style.RESET_ALL}` because all '
|
|
||||||
f'of it\'s dependencies remained unchanged')
|
|
||||||
return True
|
|
||||||
|
|
||||||
stage_values = self.cfg.get_r_env(provider.name).values
|
|
||||||
modrunctx = config_mod_runctx(provider, self.cfg.platform,
|
|
||||||
stage_values, self.dep_paths,
|
|
||||||
self.cfg.get_dependency_overrides())
|
|
||||||
module_exec(provider.module, modrunctx)
|
|
||||||
|
|
||||||
self.run_stages.discard(provider.name)
|
|
||||||
|
|
||||||
if not req_exists(paths):
|
|
||||||
raise DependencyNotProducedException(dep, provider.name)
|
|
||||||
|
|
||||||
return True
|
|
||||||
|
|
||||||
def execute(self):
|
|
||||||
self._build_dep(self.target)
|
|
||||||
if self.symbicache:
|
|
||||||
update_dep_statuses(self.dep_paths[self.target], '__target',
|
|
||||||
self.symbicache)
|
|
||||||
sfprint(0, f'Target `{Style.BRIGHT + self.target + Style.RESET_ALL}` '
|
|
||||||
f'-> {self.dep_paths[self.target]}')
|
|
||||||
|
|
||||||
def display_dep_info(stages: 'Iterable[Stage]'):
|
|
||||||
sfprint(0, 'Platform dependencies/targets:')
|
|
||||||
longest_out_name_len = 0
|
|
||||||
for stage in stages:
|
|
||||||
for out in stage.produces:
|
|
||||||
l = len(out.name)
|
|
||||||
if l > longest_out_name_len:
|
|
||||||
longest_out_name_len = l
|
|
||||||
|
|
||||||
desc_indent = longest_out_name_len + 7
|
|
||||||
nl_indentstr = '\n'
|
|
||||||
for _ in range(0, desc_indent):
|
|
||||||
nl_indentstr += ' '
|
|
||||||
|
|
||||||
for stage in stages:
|
|
||||||
for out in stage.produces:
|
|
||||||
pname = Style.BRIGHT + out.name + Style.RESET_ALL
|
|
||||||
indent = ''
|
|
||||||
for _ in range(0, desc_indent - len(pname) + 3):
|
|
||||||
indent += ' '
|
|
||||||
specstr = '???'
|
|
||||||
if out.spec == 'req':
|
|
||||||
specstr = f'{Fore.BLUE}guaranteed{Fore.RESET}'
|
|
||||||
elif out.spec == 'maybe':
|
|
||||||
specstr = f'{Fore.YELLOW}not guaranteed{Fore.RESET}'
|
|
||||||
elif out.spec == 'demand':
|
|
||||||
specstr = f'{Fore.RED}on-demand{Fore.RESET}'
|
|
||||||
pgen = f'{Style.DIM}stage: `{stage.name}`, '\
|
|
||||||
f'spec: {specstr}{Style.RESET_ALL}'
|
|
||||||
pdesc = stage.meta[out.name].replace('\n', nl_indentstr)
|
|
||||||
sfprint(0, f' {Style.BRIGHT + out.name + Style.RESET_ALL}:'
|
|
||||||
f'{indent}{pdesc}{nl_indentstr}{pgen}')
|
|
||||||
|
|
||||||
def display_stage_info(stage: Stage):
|
|
||||||
if stage is None:
|
|
||||||
sfprint(0, f'Stage does not exist')
|
|
||||||
sfbuild_fail()
|
|
||||||
return
|
|
||||||
|
|
||||||
sfprint(0, f'Stage `{Style.BRIGHT}{stage.name}{Style.RESET_ALL}`:')
|
|
||||||
sfprint(0, f' Module: `{Style.BRIGHT}{stage.module.name}{Style.RESET_ALL}`')
|
|
||||||
sfprint(0, f' Module info:')
|
|
||||||
|
|
||||||
mod_info = get_module_info(stage.module)
|
|
||||||
mod_info = '\n '.join(mod_info.split('\n'))
|
|
||||||
|
|
||||||
sfprint(0, f' {mod_info}')
|
|
||||||
|
|
||||||
sfbuild_done_str = Style.BRIGHT + Fore.GREEN + 'DONE'
|
|
||||||
sfbuild_silent = 0
|
|
||||||
|
|
||||||
def sfbuild_fail():
|
|
||||||
global sfbuild_done_str
|
|
||||||
sfbuild_done_str = Style.BRIGHT + Fore.RED + 'FAILED'
|
|
||||||
|
|
||||||
def sfbuild_done():
|
|
||||||
sfprint(1, f'sfbuild: {sfbuild_done_str}'
|
|
||||||
f'{Style.RESET_ALL + Fore.RESET}')
|
|
||||||
exit(0)
|
|
||||||
|
|
||||||
def setup_resolution_env():
|
|
||||||
""" Sets up a ResolutionEnv with sfbuild's default built-ins. """
|
|
||||||
|
|
||||||
r_env = ResolutionEnv({
|
|
||||||
'shareDir': share_dir_path,
|
|
||||||
'binDir': os.path.realpath(os.path.join(share_dir_path, '../../bin'))
|
|
||||||
})
|
|
||||||
r_env.add_values(sf_ugly.generate_values())
|
|
||||||
return r_env
|
|
||||||
|
|
||||||
def open_project_flow_config(path: str) -> ProjectFlowConfig:
|
|
||||||
try:
|
|
||||||
flow_cfg = open_project_flow_cfg(path)
|
|
||||||
except FileNotFoundError as _:
|
|
||||||
fatal(-1, 'The provided flow configuration file does not exist')
|
|
||||||
return flow_cfg
|
|
||||||
|
|
||||||
def verify_platform_stage_params(flow_cfg: FlowConfig,
|
|
||||||
platform: 'str | None' = None,
|
|
||||||
stage: 'str | None' = None):
|
|
||||||
if platform:
|
|
||||||
if not verify_platform_name(platform, mypath):
|
|
||||||
sfprint(0, f'Platform `{platform}`` is unsupported.')
|
|
||||||
return False
|
|
||||||
if args.platform not in flow_cfg.platforms():
|
|
||||||
sfprint(0, f'Platform `{platform}`` is not in project.')
|
|
||||||
return False
|
|
||||||
|
|
||||||
if stage:
|
|
||||||
if not verify_stage(platform, stage, mypath):
|
|
||||||
sfprint(0, f'Stage `{stage}` is invalid.')
|
|
||||||
sfbuild_fail()
|
|
||||||
return False
|
|
||||||
|
|
||||||
return True
|
|
||||||
|
|
||||||
def get_platform_name_for_part(part_name: str):
|
|
||||||
"""
|
|
||||||
Gets a name that identifies the platform setup required for a specific chip.
|
|
||||||
The reason for such distinction is that plenty of chips with different names
|
|
||||||
differ only in a type of package they use.
|
|
||||||
"""
|
|
||||||
|
|
||||||
d: dict
|
|
||||||
with open(os.path.join(mypath, 'part_db/parts.json')) as f:
|
|
||||||
d = json.loads(f.read())
|
|
||||||
return d.get(part_name.upper())
|
|
||||||
|
|
||||||
def cmd_build(args: Namespace):
|
|
||||||
""" sfbuild's `build` command implementation """
|
|
||||||
|
|
||||||
project_flow_cfg: ProjectFlowConfig = None
|
|
||||||
|
|
||||||
|
|
||||||
platform = args.platform
|
|
||||||
if platform is None:
|
|
||||||
if args.part:
|
|
||||||
platform = get_platform_name_for_part(args.part)
|
|
||||||
|
|
||||||
if args.flow:
|
|
||||||
project_flow_cfg = open_project_flow_config(args.flow)
|
|
||||||
elif platform is not None:
|
|
||||||
project_flow_cfg = ProjectFlowConfig('.temp.flow.json')
|
|
||||||
project_flow_cfg.flow_cfg = get_cli_flow_config(args, platform)
|
|
||||||
if platform is None and project_flow_cfg is not None:
|
|
||||||
platform = project_flow_cfg.get_default_platform()
|
|
||||||
if platform is None:
|
|
||||||
fatal(-1, 'You have to specify a platform name or a part name or '
|
|
||||||
'configure a default platform.')
|
|
||||||
if platform is None or project_flow_cfg is None:
|
|
||||||
fatal(-1, 'No configuration was provided. Use `--flow`, `--platform` or '
|
|
||||||
'`--part` to configure flow..')
|
|
||||||
|
|
||||||
platform_path = os.path.join(mypath, 'platforms', platform + '.json')
|
|
||||||
platform_def = None
|
|
||||||
try:
|
|
||||||
with open(platform_path) as platform_file:
|
|
||||||
platform_def = platform_file.read()
|
|
||||||
except FileNotFoundError as _:
|
|
||||||
fatal(-1, f'The platform flow definition file {platform_path} for the platform '
|
|
||||||
f'{platform} referenced in flow definition file {args.flow} '
|
|
||||||
'cannot be found.')
|
|
||||||
|
|
||||||
r_env = setup_resolution_env()
|
|
||||||
|
|
||||||
sfprint(2, 'Scanning modules...')
|
|
||||||
scan_modules(mypath)
|
|
||||||
|
|
||||||
flow_definition_dict = json.loads(platform_def)
|
|
||||||
flow_def = FlowDefinition(flow_definition_dict, r_env)
|
|
||||||
flow_cfg = FlowConfig(project_flow_cfg, flow_def, platform)
|
|
||||||
|
|
||||||
|
|
||||||
if len(flow_cfg.stages) == 0:
|
|
||||||
fatal(-1, 'Platform flow does not define any stage')
|
|
||||||
|
|
||||||
if args.info:
|
|
||||||
display_dep_info(flow_cfg.stages.values())
|
|
||||||
sfbuild_done()
|
|
||||||
|
|
||||||
if args.stageinfo:
|
|
||||||
display_stage_info(flow_cfg.stages.get(args.stageinfo[0]))
|
|
||||||
sfbuild_done()
|
|
||||||
|
|
||||||
target = args.target
|
|
||||||
if target is None:
|
|
||||||
target = project_flow_cfg.get_default_target(platform)
|
|
||||||
if target is None:
|
|
||||||
fatal(-1, 'Please specify desired target using `--target` option '
|
|
||||||
'or configure a default target.')
|
|
||||||
|
|
||||||
flow = Flow(
|
|
||||||
target=target,
|
|
||||||
cfg=flow_cfg,
|
|
||||||
symbicache=SymbiCache(SYMBICACHEPATH) if not args.nocache else None
|
|
||||||
)
|
|
||||||
|
|
||||||
dep_print_verbosity = 0 if args.pretend else 2
|
|
||||||
sfprint(dep_print_verbosity, '\nProject status:')
|
|
||||||
flow.print_resolved_dependencies(dep_print_verbosity)
|
|
||||||
sfprint(dep_print_verbosity, '')
|
|
||||||
|
|
||||||
if args.pretend:
|
|
||||||
sfbuild_done()
|
|
||||||
|
|
||||||
try:
|
|
||||||
flow.execute()
|
|
||||||
except Exception as e:
|
|
||||||
sfprint(0, e)
|
|
||||||
sfbuild_fail()
|
|
||||||
|
|
||||||
if flow.symbicache:
|
|
||||||
flow.symbicache.save()
|
|
||||||
|
|
||||||
def cmd_show_dependencies(args: Namespace):
|
|
||||||
""" sfbuild's `showd` command implementation """
|
|
||||||
|
|
||||||
flow_cfg = open_project_flow_config(args.flow)
|
|
||||||
|
|
||||||
if not verify_platform_stage_params(flow_cfg, args.platform):
|
|
||||||
sfbuild_fail()
|
|
||||||
return
|
|
||||||
|
|
||||||
platform_overrides: 'set | None' = None
|
|
||||||
if args.platform is not None:
|
|
||||||
platform_overrides = \
|
|
||||||
set(flow_cfg.get_dependency_platform_overrides(args.platform).keys())
|
|
||||||
|
|
||||||
display_list = []
|
|
||||||
|
|
||||||
raw_deps = flow_cfg.get_dependencies_raw(args.platform)
|
|
||||||
|
|
||||||
for dep_name, dep_paths in raw_deps.items():
|
|
||||||
prstr: str
|
|
||||||
if (platform_overrides is not None) and (dep_name in platform_overrides):
|
|
||||||
prstr = f'{Style.DIM}({args.platform}){Style.RESET_ALL} ' \
|
|
||||||
f'{Style.BRIGHT + dep_name + Style.RESET_ALL}: {dep_paths}'
|
|
||||||
else:
|
|
||||||
prstr = f'{Style.BRIGHT + dep_name + Style.RESET_ALL}: {dep_paths}'
|
|
||||||
|
|
||||||
display_list.append((dep_name, prstr))
|
|
||||||
|
|
||||||
display_list.sort(key = lambda p: p[0])
|
|
||||||
|
|
||||||
for _, prstr in display_list:
|
|
||||||
sfprint(0, prstr)
|
|
||||||
|
|
||||||
set_verbosity_level(-1)
|
|
||||||
|
|
||||||
def main():
|
|
||||||
parser = setup_argparser()
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
set_verbosity_level(args.verbose - (1 if args.silent else 0))
|
|
||||||
|
|
||||||
if args.command == 'build':
|
|
||||||
cmd_build(args)
|
|
||||||
sfbuild_done()
|
|
||||||
|
|
||||||
if args.command == 'showd':
|
|
||||||
cmd_show_dependencies(args)
|
|
||||||
sfbuild_done()
|
|
||||||
|
|
||||||
sfprint(0, 'Please use a command.\nUse `--help` flag to learn more.')
|
|
||||||
sfbuild_done()
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
Loading…
Reference in New Issue