Hardware
/
f4pga
mirror of https://github.com/chipsalliance/f4pga.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Rename symbicache to f4cache 

Signed-off-by: Krzysztof Boronski <kboronski@antmicro.com>
This commit is contained in:
Krzysztof Boronski 2022-05-27 10:15:12 -05:00
parent 9a3ceba5c1
commit 47733138a3
4 changed files with 42 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/f4pga/DevNotes.md
View File

@ -58,8 +58,8 @@ particular consumer. This is necessary, because the system tries to avoid rebuil
when possible and status of each file (modified/unmodified) may differ in regards when possible and status of each file (modified/unmodified) may differ in regards
to individual stages. to individual stages.
Keeping track of status of each file is done using `SymbiCache` class, which is Keeping track of status of each file is done using `F4Cache` class, which is
defined in `sf_cache.py` file. `SymbiCache` is used mostly inside `Flow`'s methods. defined in `cache.py` file. `F4Cache` is used mostly inside `Flow`'s methods.
### Internal environmental variable system ### Internal environmental variable system

8
docs/f4pga/Usage.md
View File

@ -71,21 +71,21 @@ file or derived from the paths of the dependencies taken by the module.
A *flow* is set of *modules* executed in a right order to produce a *target*. A *flow* is set of *modules* executed in a right order to produce a *target*.
### .symbicache ### .f4cache
All *dependencies* are tracked by a modification tracking system which stores hashes of the files All *dependencies* are tracked by a modification tracking system which stores hashes of the files
(directories get always `'0'` hash) in `.symbicache` file in the root of the project. (directories get always `'0'` hash) in `.f4cache` file in the root of the project.
When F4PGA constructs a *flow*, it will try to omit execution of modules which would receive the same data on their When F4PGA constructs a *flow*, it will try to omit execution of modules which would receive the same data on their
input. input.
There is a strong _assumption_ there that a *module*'s output remains unchanged if the input configuration isn't There is a strong _assumption_ there that a *module*'s output remains unchanged if the input configuration isn't
changed, ie. *modules* are deterministic. This is might be not true for some tools and in case you really want to re-run changed, ie. *modules* are deterministic. This is might be not true for some tools and in case you really want to re-run
a stage, there's a `--nocache` option that treats the `.symbicache` file as if it was empty. a stage, there's a `--nocache` option that treats the `.f4cache` file as if it was empty.
### Resolution ### Resolution
A *dependency* is said to be *resolved* if it meets one of the following criteria: A *dependency* is said to be *resolved* if it meets one of the following criteria:
* it exists on persistent storage and its hash matches the one stored in .symbicache * it exists on persistent storage and its hash matches the one stored in .f4cache
* there exists such *flow* that all of the dependencies of its modules are *resolved* and it produces the *dependency* in * there exists such *flow* that all of the dependencies of its modules are *resolved* and it produces the *dependency* in
question. question.

59
f4pga/__init__.py
View File

@ -35,7 +35,7 @@ from f4pga.common import (
sub as common_sub sub as common_sub
) )
from f4pga.module import * from f4pga.module import *
from f4pga.cache import SymbiCache from f4pga.cache import F4Cache
from f4pga.flow_config import ( from f4pga.flow_config import (
ProjectFlowConfig, ProjectFlowConfig,
FlowConfig, FlowConfig,
@ -48,7 +48,7 @@ from f4pga.module_inspector import get_module_info
from f4pga.stage import Stage from f4pga.stage import Stage
from f4pga.argparser import setup_argparser, get_cli_flow_config from f4pga.argparser import setup_argparser, get_cli_flow_config
SYMBICACHEPATH = '.symbicache' F4CACHEPATH = '.f4cache'
binpath = str(Path(sys_argv[0]).resolve().parent.parent) binpath = str(Path(sys_argv[0]).resolve().parent.parent)
mypath = str(Path(__file__).resolve().parent) mypath = str(Path(__file__).resolve().parent)
@ -109,9 +109,9 @@ def map_outputs_to_stages(stages: 'list[Stage]'):
'provider at most.') 'provider at most.')
return os_map return os_map
def filter_existing_deps(deps: 'dict[str, ]', symbicache): def filter_existing_deps(deps: 'dict[str, ]', f4cache):
return [(n, p) for n, p in deps.items() \ return [(n, p) for n, p in deps.items() \
if req_exists(p)] # and not dep_differ(p, symbicache)] if req_exists(p)] # and not dep_differ(p, f4cache)]
def get_stage_values_override(og_values: dict, stage: Stage): def get_stage_values_override(og_values: dict, stage: Stage):
values = og_values.copy() values = og_values.copy()
@ -144,35 +144,34 @@ def prepare_stage_input(stage: Stage, values: dict, dep_paths: 'dict[str, ]',
return stage_mod_cfg return stage_mod_cfg
def update_dep_statuses(paths, consumer: str, symbicache: SymbiCache): def update_dep_statuses(paths, consumer: str, f4cache: F4Cache):
if type(paths) is str: if type(paths) is str:
return symbicache.update(Path(paths), consumer) return f4cache.update(Path(paths), consumer)
elif type(paths) is list: elif type(paths) is list:
for p in paths: for p in paths:
return update_dep_statuses(p, consumer, symbicache) return update_dep_statuses(p, consumer, f4cache)
elif type(paths) is dict: elif type(paths) is dict:
for _, p in paths.items(): for _, p in paths.items():
return update_dep_statuses(p, consumer, symbicache) return update_dep_statuses(p, consumer, f4cache)
fatal(-1, 'WRONG PATHS TYPE') fatal(-1, 'WRONG PATHS TYPE')
def dep_differ(paths, consumer: str, symbicache: SymbiCache): def dep_differ(paths, consumer: str, f4cache: F4Cache):
""" """
Check if a dependency differs from its last version, lack of dependency is Check if a dependency differs from its last version, lack of dependency is
treated as "differs" treated as "differs"
""" """
if type(paths) is str: if type(paths) is str:
s = symbicache.get_status(paths, consumer) return f4cache.get_status(paths, consumer) != 'same'
return symbicache.get_status(paths, consumer) != 'same'
elif type(paths) is list: elif type(paths) is list:
return True in [dep_differ(p, consumer, symbicache) for p in paths] return True in [dep_differ(p, consumer, f4cache) for p in paths]
elif type(paths) is dict: elif type(paths) is dict:
return True in [dep_differ(p, consumer, symbicache) \ return True in [dep_differ(p, consumer, f4cache) \
for _, p in paths.items()] for _, p in paths.items()]
return False return False
def dep_will_differ(target: str, paths, consumer: str, def dep_will_differ(target: str, paths, consumer: str,
os_map: 'dict[str, Stage]', run_stages: 'set[str]', os_map: 'dict[str, Stage]', run_stages: 'set[str]',
symbicache: SymbiCache): f4cache: F4Cache):
""" """
Check if a dependency or any of the dependencies it depends on differ from Check if a dependency or any of the dependencies it depends on differ from
their last versions. their last versions.
@ -181,8 +180,8 @@ def dep_will_differ(target: str, paths, consumer: str,
provider = os_map.get(target) provider = os_map.get(target)
if provider: if provider:
return (provider.name in run_stages) or \ return (provider.name in run_stages) or \
dep_differ(paths, consumer, symbicache) dep_differ(paths, consumer, f4cache)
return dep_differ(paths, consumer, symbicache) return dep_differ(paths, consumer, f4cache)
def _print_unreachable_stage_message(provider: Stage, take: str): def _print_unreachable_stage_message(provider: Stage, take: str):
sfprint(0, ' Stage ' sfprint(0, ' Stage '
@ -214,31 +213,31 @@ class Flow:
run_stages: 'set[str]' run_stages: 'set[str]'
# Number of stages that relied on outdated version of a (checked) dependency # Number of stages that relied on outdated version of a (checked) dependency
deps_rebuilds: 'dict[str, int]' deps_rebuilds: 'dict[str, int]'
symbicache: 'SymbiCache | None' f4cache: 'F4Cache | None'
flow_cfg: FlowConfig flow_cfg: FlowConfig
def __init__(self, target: str, cfg: FlowConfig, def __init__(self, target: str, cfg: FlowConfig,
symbicache: 'SymbiCache | None'): f4cache: 'F4Cache | None'):
self.target = target self.target = target
self.os_map = map_outputs_to_stages(cfg.stages.values()) self.os_map = map_outputs_to_stages(cfg.stages.values())
explicit_deps = cfg.get_dependency_overrides() explicit_deps = cfg.get_dependency_overrides()
# print(explicit_deps) # print(explicit_deps)
self.dep_paths = dict(filter_existing_deps(explicit_deps, symbicache)) self.dep_paths = dict(filter_existing_deps(explicit_deps, f4cache))
self.run_stages = set() self.run_stages = set()
self.symbicache = symbicache self.f4cache = f4cache
self.cfg = cfg self.cfg = cfg
self.deps_rebuilds = {} self.deps_rebuilds = {}
self._resolve_dependencies(self.target, set()) self._resolve_dependencies(self.target, set())
def _dep_will_differ(self, dep: str, paths, consumer: str): def _dep_will_differ(self, dep: str, paths, consumer: str):
if not self.symbicache: # Handle --nocache mode if not self.f4cache: # Handle --nocache mode
return True return True
return dep_will_differ(dep, paths, consumer, return dep_will_differ(dep, paths, consumer,
self.os_map, self.run_stages, self.os_map, self.run_stages,
self.symbicache) self.f4cache)
def _resolve_dependencies(self, dep: str, stages_checked: 'set[str]'): def _resolve_dependencies(self, dep: str, stages_checked: 'set[str]'):
# Initialize the dependency status if necessary # Initialize the dependency status if necessary
@ -353,16 +352,16 @@ class Flow:
else: else:
assert(provider) assert(provider)
any_dep_differ = False if (self.symbicache is not None) else True any_dep_differ = False if (self.f4cache is not None) else True
for p_dep in provider.takes: for p_dep in provider.takes:
if not self._build_dep(p_dep.name): if not self._build_dep(p_dep.name):
assert (p_dep.spec != 'req') assert (p_dep.spec != 'req')
continue continue
if self.symbicache is not None: if self.f4cache is not None:
any_dep_differ |= \ any_dep_differ |= \
update_dep_statuses(self.dep_paths[p_dep.name], update_dep_statuses(self.dep_paths[p_dep.name],
provider.name, self.symbicache) provider.name, self.f4cache)
# If dependencies remained the same, consider the dep as up-to date # If dependencies remained the same, consider the dep as up-to date
# For example, when changing a comment in Verilog source code, # For example, when changing a comment in Verilog source code,
@ -390,9 +389,9 @@ class Flow:
def execute(self): def execute(self):
self._build_dep(self.target) self._build_dep(self.target)
if self.symbicache: if self.f4cache:
update_dep_statuses(self.dep_paths[self.target], '__target', update_dep_statuses(self.dep_paths[self.target], '__target',
self.symbicache) self.f4cache)
sfprint(0, f'Target `{Style.BRIGHT + self.target + Style.RESET_ALL}` ' sfprint(0, f'Target `{Style.BRIGHT + self.target + Style.RESET_ALL}` '
f'-> {self.dep_paths[self.target]}') f'-> {self.dep_paths[self.target]}')
@ -583,7 +582,7 @@ def cmd_build(args: Namespace):
flow = Flow( flow = Flow(
target=target, target=target,
cfg=flow_cfg, cfg=flow_cfg,
symbicache=SymbiCache(SYMBICACHEPATH) if not args.nocache else None f4cache=F4Cache(F4CACHEPATH) if not args.nocache else None
) )
dep_print_verbosity = 0 if args.pretend else 2 dep_print_verbosity = 0 if args.pretend else 2
@ -600,8 +599,8 @@ def cmd_build(args: Namespace):
sfprint(0, e) sfprint(0, e)
sfbuild_fail() sfbuild_fail()
if flow.symbicache: if flow.f4cache:
flow.symbicache.save() flow.f4cache.save()
def cmd_show_dependencies(args: Namespace): def cmd_show_dependencies(args: Namespace):
""" sfbuild's `showd` command implementation """ """ sfbuild's `showd` command implementation """

14
f4pga/cache.py
View File

@ -4,9 +4,9 @@ from json import dump as json_dump, load as json_load, JSONDecodeError
from f4pga.common import sfprint from f4pga.common import sfprint
class SymbiCache: class F4Cache:
""" """
`SymbiCache` is used to track changes among dependencies and keep the status of the files on a persistent storage. `F4Cache` is used to track changes among dependencies and keep the status of the files on a persistent storage.
Files which are tracked get their checksums calculated and stored in a file. Files which are tracked get their checksums calculated and stored in a file.
If file's checksum differs from the one saved in a file, that means, the file has changed. If file's checksum differs from the one saved in a file, that means, the file has changed.
""" """
@ -91,13 +91,13 @@ class SymbiCache:
try: try:
with Path(self.cachefile_path).open('r') as rfptr: with Path(self.cachefile_path).open('r') as rfptr:
self.hashes = json_load(rfptr) self.hashes = json_load(rfptr)
except JSONDecodeError as jerr: except JSONDecodeError:
sfprint(0, 'WARNING: .symbicache is corrupted!\n' sfprint(0, f'WARNING: `{self.cachefile_path}` f4cache is corrupted!\n'
'This will cause flow to re-execute from the beginning.') 'This will cause flow to re-execute from the beginning.')
self.hashes = {} self.hashes = {}
except FileNotFoundError: except FileNotFoundError:
sfprint(0, 'Couldn\'t open .symbicache cache file.\n' sfprint(0, f'Couldn\'t open `{self.cachefile_path}` cache file.\n'
'This will cause flow to re-execute from the beginning.') 'This will cause flow to re-execute from the beginning.')
self.hashes = {} self.hashes = {}
def save(self): def save(self):