Coverage for tsfpga/git_simulation_subset.py: 80%
105 statements
« prev ^ index » next coverage.py v7.6.4, created at 2024-10-28 20:52 +0000
« prev ^ index » next coverage.py v7.6.4, created at 2024-10-28 20:52 +0000
1# --------------------------------------------------------------------------------------------------
2# Copyright (c) Lukas Vik. All rights reserved.
3#
4# This file is part of the tsfpga project, a project platform for modern FPGA development.
5# https://tsfpga.com
6# https://github.com/tsfpga/tsfpga
7# --------------------------------------------------------------------------------------------------
9# Standard libraries
10import re
11from collections.abc import Iterable
12from pathlib import Path
13from typing import TYPE_CHECKING, Any, Optional
15# Third party libraries
16from git.repo import Repo
18# Local folder libraries
19from .hdl_file import HdlFile
21if TYPE_CHECKING:
22 # Local folder libraries
23 from .module import BaseModule
24 from .module_list import ModuleList
26VHDL_FILE_ENDINGS = HdlFile.file_endings_mapping[HdlFile.Type.VHDL]
29class GitSimulationSubset:
30 """
31 Find a subset of testbenches to simulate based on git history.
32 """
34 # Should work with
35 # * tb_<name>.vhd
36 # * <name>_tb.vhd
37 # and possibly .vhdl as file extension.
38 _re_tb_filename = re.compile(r"^(tb_.+\.vhdl?)|(.+\_tb.vhdl?)$")
39 # Should work with .toml, .json, .yaml, etc.
40 _re_register_data_filename = re.compile(r"^regs_(.+)\.[a-z]+$")
42 def __init__(
43 self,
44 repo_root: Path,
45 reference_branch: str,
46 vunit_proj: Any,
47 modules: Optional["ModuleList"] = None,
48 vunit_preprocessed_path: Optional[Path] = None,
49 ) -> None:
50 """
51 Arguments:
52 repo_root: Root directory where git commands will be run.
53 reference_branch: What git branch to compare against, when finding what files have
54 changed. Typically "origin/main" or "origin/master".
55 vunit_proj: A vunit project with all source files and testbenches added. Will be used
56 for dependency scanning.
57 modules: A list of modules that are included in the VUnit project.
59 When this argument is provided, this class will look for changes in the modules'
60 register data files, and simulate the testbenches that depend on register artifacts
61 in case of any changes.
63 This argument **must** be supplied if VUnit preprocessing is enabled.
64 vunit_preprocessed_path: If location/check preprocessing is enabled
65 in your VUnit project, supply the path to ``vunit_out/preprocessed``.
66 """
67 self._repo_root = repo_root
68 self._reference_branch = reference_branch
69 self._vunit_proj = vunit_proj
70 self._modules = modules
71 self._vunit_preprocessed_path = vunit_preprocessed_path
73 def find_subset(self) -> list[tuple[str, str]]:
74 """
75 Return all testbenches that have changes, or depend on files that have changes.
77 Return:
78 The testbench names and their corresponding library names.
79 A list of tuples ("testbench name", "library name").
80 """
81 diff_files = self._find_diff_vhd_files()
83 if self._vunit_preprocessed_path:
84 # If preprocessing is enabled, VUnit's dependency graph is based on the files that
85 # are in the vunit_out/preprocessed folder, not in the file's original location.
86 # So manipulate the paths to point there.
87 diff_files = self._get_preprocessed_file_locations(diff_files)
88 self._print_file_list("Resolved diff file locations to be", diff_files)
90 # Find all testbench files that are available
91 testbenches = self._find_testbenches()
93 # Gather the testbenches that depend on any files that have diffs
94 testbenches_to_run = []
95 for testbench_source_file, library_name in testbenches:
96 if self._source_file_depends_on_files(
97 source_file=testbench_source_file,
98 files=diff_files,
99 ):
100 testbench_file_name = Path(testbench_source_file.name).stem
101 testbenches_to_run.append((testbench_file_name, library_name))
103 return testbenches_to_run
105 def _find_diff_vhd_files(self) -> set[Path]:
106 repo = Repo(self._repo_root)
108 head_commit = repo.head.commit
109 reference_commit = repo.commit(self._reference_branch)
111 # Local uncommitted changed
112 working_tree_changes = head_commit.diff(None)
114 # Changes in the git log compared to the reference commit
115 history_changes = head_commit.diff(reference_commit)
117 return self._iterate_vhd_file_diffs(diffs=working_tree_changes + history_changes)
119 def _iterate_vhd_file_diffs(self, diffs: Any) -> set[Path]:
120 """
121 Return the currently existing VHDL files that have been changed (added/renamed/modified)
122 within any of the ``diffs`` commits.
124 Will also try to find VHDL files that depend on generated register artifacts that
125 have changed.
126 """
127 files = set()
129 def add_register_artifacts_if_match(
130 diff_path: Path, module_register_data_file: Path, module: "BaseModule"
131 ) -> None:
132 """
133 Note that Path.__eq__ does not do normalization of paths.
134 If one argument is a relative path and the other is an absolute path, they will not
135 be considered equal.
136 Hence, it is important that both paths are resolved before comparison.
137 """
138 if diff_path != module_register_data_file:
139 return
141 re_match = self._re_register_data_filename.match(module_register_data_file.name)
142 assert (
143 re_match is not None
144 ), "Register data file does not use the expected naming convention"
146 register_list_name = re_match.group(1)
147 regs_pkg_path = module.register_synthesis_folder / f"{register_list_name}_regs_pkg.vhd"
149 # It is okay to add only the base register package, since all other
150 # register artifacts depend on it.
151 # This file will typically not exist yet in a CI flow, so it doesn't make sense to
152 # assert for its existence.
153 files.add(regs_pkg_path)
155 for diff_path in self._iterate_diff_paths(diffs=diffs):
156 if diff_path.name.endswith(VHDL_FILE_ENDINGS):
157 files.add(diff_path)
159 elif self._modules is not None:
160 for module in self._modules:
161 module_register_data_file = module.register_data_file
163 if isinstance(module_register_data_file, list):
164 # In users implement a sub-class of BaseModule that has multiple register
165 # lists. This is not a standard use case, but we support it here.
166 for data_file in module_register_data_file:
167 add_register_artifacts_if_match(
168 diff_path=diff_path,
169 module_register_data_file=data_file,
170 module=module,
171 )
173 else:
174 add_register_artifacts_if_match(
175 diff_path=diff_path,
176 module_register_data_file=module_register_data_file,
177 module=module,
178 )
180 self._print_file_list("Found git diff in the following files", files)
181 return files
183 def _iterate_diff_paths(self, diffs: Any) -> Iterable[Path]:
184 for diff in diffs:
185 # The diff contains "a" -> "b" changes information. In case of file deletion, a_path
186 # will be set but not b_path. Removed files are not included by this method.
187 if diff.b_path is not None:
188 b_path = Path(diff.b_path)
190 # A file can be changed in an early commit, but then removed/renamed in a
191 # later commit. Include only files that are currently existing.
192 if b_path.exists():
193 yield b_path.resolve()
195 def _get_preprocessed_file_locations(self, vhd_files: set[Path]) -> set[Path]:
196 """
197 Find the location of a VUnit preprocessed file, based on the path in the modules tree.
198 Not all VHDL files are included in the simulation projects (e.g. often files that depend
199 on IP cores are excluded), hence files that can not be found in any module's simulation
200 files are ignored.
201 """
202 assert (
203 self._modules is not None
204 ), "Modules must be supplied when VUnit preprocessing is enabled"
206 result = set()
207 for vhd_file in vhd_files:
208 library_name = self._get_library_name_from_path(vhd_file)
210 if library_name is not None:
211 # Ignore that '_vunit_preprocessed_path' is type 'Path | None', since we only come
212 # if it is not 'None'.
213 preprocessed_file = (
214 self._vunit_preprocessed_path # type: ignore[operator]
215 / library_name
216 / vhd_file.name
217 )
218 assert preprocessed_file.exists(), preprocessed_file
220 result.add(preprocessed_file)
222 return result
224 def _get_library_name_from_path(self, vhd_file: Path) -> Optional[str]:
225 """
226 Returns (str): Library name for the given file path.
227 Will return None if no library can be found.
228 """
229 # Ignore that '_modules' is type 'Path | None', since we only come
230 # if it is not 'None'.
231 for module in self._modules: # type: ignore[union-attr]
232 for module_hdl_file in module.get_simulation_files(include_ip_cores=True):
233 if module_hdl_file.path.name == vhd_file.name:
234 return module.library_name
236 print(f"Could not find library for file {vhd_file}. It will be skipped.")
237 return None
239 def _find_testbenches(self) -> list[tuple[Any, str]]:
240 """
241 Find all testbench files that are available in the VUnit project.
243 Return:
244 The VUnit ``SourceFile`` objects and library names for the files.
245 """
246 result = []
247 for source_file in self._vunit_proj.get_source_files():
248 source_file_path = Path(source_file.name)
249 assert source_file_path.exists(), source_file_path
251 # The file is considered a testbench if it follows the tb naming pattern
252 if self._re_tb_filename.match(source_file_path.name) is not None:
253 result.append((source_file, source_file.library.name))
255 return result
257 def _source_file_depends_on_files(self, source_file: Any, files: set[Path]) -> bool:
258 """
259 Return True if the source_file depends on any of the files.
260 """
261 # Note that this includes the source_file itself. Is a list of SourceFile objects.
262 implementation_subset = self._vunit_proj.get_implementation_subset([source_file])
264 # Convert to a set of absolute Paths, for comparison with "files" which is of that type.
265 source_file_dependencies = {
266 Path(implementation_file.name).resolve()
267 for implementation_file in implementation_subset
268 }
270 intersection = source_file_dependencies & files
271 if not intersection:
272 return False
274 self._print_file_list(
275 f"Testbench {source_file.name} depends on the following files which have a diff",
276 intersection,
277 )
278 return True
280 @staticmethod
281 def _print_file_list(title: str, files: set[Path]) -> None:
282 print(f"{title}:")
283 for file_path in files:
284 print(f" {file_path}")
285 print()