Coverage for tsfpga/git_simulation_subset.py: 78%
111 statements
« prev ^ index » next coverage.py v7.6.10, created at 2025-01-21 20:51 +0000
1# --------------------------------------------------------------------------------------------------
2# Copyright (c) Lukas Vik. All rights reserved.
3#
4# This file is part of the tsfpga project, a project platform for modern FPGA development.
5# https://tsfpga.com
6# https://github.com/tsfpga/tsfpga
7# --------------------------------------------------------------------------------------------------
9# Standard libraries
10import re
11from collections.abc import Iterable
12from pathlib import Path
13from typing import TYPE_CHECKING, Any, Optional
15# Third party libraries
16from git.repo import Repo
18# Local folder libraries
19from .hdl_file import HdlFile
21if TYPE_CHECKING:
22 # Third party libraries
23 from git.diff import DiffIndex
24 from vunit.ui import VUnit
26 # Local folder libraries
27 from .module import BaseModule
28 from .module_list import ModuleList
30VHDL_FILE_ENDINGS = HdlFile.file_endings_mapping[HdlFile.Type.VHDL]
33class GitSimulationSubset:
34 """
35 Find a subset of testbenches to simulate based on git history.
36 """
38 # Should work with
39 # * tb_<name>.vhd
40 # * <name>_tb.vhd
41 # and possibly .vhdl as file extension.
42 _re_tb_filename = re.compile(r"^(tb_.+\.vhdl?)|(.+\_tb.vhdl?)$")
43 # Should work with .toml, .json, .yaml, etc.
44 _re_register_data_filename = re.compile(r"^regs_(.+)\.[a-z]+$")
46 def __init__(
47 self,
48 repo_root: Path,
49 reference_branch: str,
50 vunit_proj: "VUnit",
51 modules: Optional["ModuleList"] = None,
52 vunit_preprocessed_path: Optional[Path] = None,
53 ) -> None:
54 """
55 Arguments:
56 repo_root: Root directory where git commands will be run.
57 reference_branch: What git branch to compare against, when finding what files have
58 changed. Typically "origin/main" or "origin/master".
59 vunit_proj: A vunit project with all source files and testbenches added. Will be used
60 for dependency scanning.
61 modules: A list of modules that are included in the VUnit project.
63 When this argument is provided, this class will look for changes in the modules'
64 register data files, and simulate the testbenches that depend on register artifacts
65 in case of any changes.
67 This argument **must** be supplied if VUnit preprocessing is enabled.
68 vunit_preprocessed_path: If location/check preprocessing is enabled
69 in your VUnit project, supply the path to ``vunit_out/preprocessed``.
70 """
71 self._repo_root = repo_root
72 self._reference_branch = reference_branch
73 self._vunit_proj = vunit_proj
74 self._modules = modules
75 self._vunit_preprocessed_path = vunit_preprocessed_path
77 def find_subset(self) -> list[tuple[str, str]]:
78 """
79 Return all testbenches that have changes, or depend on files that have changes.
81 Return:
82 The testbench names and their corresponding library names.
83 A list of tuples ("testbench name", "library name").
84 """
85 diff_files = self._find_diff_vhd_files()
87 if self._vunit_preprocessed_path:
88 # If preprocessing is enabled, VUnit's dependency graph is based on the files that
89 # are in the vunit_out/preprocessed folder, not in the file's original location.
90 # So manipulate the paths to point there.
91 diff_files = self._get_preprocessed_file_locations(diff_files)
92 self._print_file_list("Resolved diff file locations to be", diff_files)
94 # Find all testbench files that are available
95 testbenches = self._find_testbenches()
97 # Gather the testbenches that depend on any files that have diffs
98 testbenches_to_run = []
99 for testbench_source_file, library_name in testbenches:
100 if self._source_file_depends_on_files(
101 source_file=testbench_source_file,
102 files=diff_files,
103 ):
104 testbench_file_name = Path(testbench_source_file.name).stem
105 testbenches_to_run.append((testbench_file_name, library_name))
107 return testbenches_to_run
109 def _find_diff_vhd_files(self) -> set[Path]:
110 repo = Repo(self._repo_root)
112 head_commit = repo.head.commit
113 reference_commit = repo.commit(self._reference_branch)
115 # Local uncommitted changed
116 working_tree_changes = head_commit.diff(None)
118 # Changes in the git log compared to the reference commit
119 history_changes = head_commit.diff(reference_commit)
121 all_changes: "DiffIndex[Any]" = (
122 working_tree_changes + history_changes # type: ignore[assignment]
123 )
125 return self._get_vhd_files(diffs=all_changes)
127 def _get_vhd_files(self, diffs: "DiffIndex[Any]") -> set[Path]:
128 """
129 Return VHDL files that have been changed (added/renamed/modified/deleted) within any
130 of the ``diffs`` commits.
132 Will also try to find VHDL files that depend on generated register artifacts that
133 have changed.
134 """
135 files = set()
137 def add_register_artifacts_if_match(
138 diff_path: Path, module_register_data_file: Path, module: "BaseModule"
139 ) -> None:
140 """
141 Note that Path.__eq__ does not do normalization of paths.
142 If one argument is a relative path and the other is an absolute path, they will not
143 be considered equal.
144 Hence, it is important that both paths are resolved before comparison.
145 """
146 if diff_path != module_register_data_file:
147 return
149 re_match = self._re_register_data_filename.match(module_register_data_file.name)
150 assert (
151 re_match is not None
152 ), "Register data file does not use the expected naming convention"
154 register_list_name = re_match.group(1)
155 regs_pkg_path = module.register_synthesis_folder / f"{register_list_name}_regs_pkg.vhd"
157 # It is okay to add only the base register package, since all other
158 # register artifacts depend on it.
159 # This file will typically not exist yet in a CI flow, so it doesn't make sense to
160 # assert for its existence.
161 files.add(regs_pkg_path)
163 for diff_path in self._iterate_diff_paths(diffs=diffs):
164 if diff_path.name.endswith(VHDL_FILE_ENDINGS):
165 files.add(diff_path)
167 elif self._modules is not None:
168 for module in self._modules:
169 module_register_data_file = module.register_data_file
171 if isinstance(module_register_data_file, list):
172 # In case users implement a sub-class of BaseModule that has multiple
173 # register lists.
174 # This is not a standard use case that we recommend or support in general,
175 # but we support it here for convenience.
176 for data_file in module_register_data_file:
177 add_register_artifacts_if_match(
178 diff_path=diff_path,
179 module_register_data_file=data_file,
180 module=module,
181 )
183 else:
184 add_register_artifacts_if_match(
185 diff_path=diff_path,
186 module_register_data_file=module_register_data_file,
187 module=module,
188 )
190 self._print_file_list("Found git diff related to the following files", files)
191 return files
193 def _iterate_diff_paths(self, diffs: "DiffIndex[Any]") -> Iterable[Path]:
194 """
195 * If a file is modified, ``a_path`` and ``b_path`` are set and point to the same file.
196 * If a file is added, ``a_path`` is None and ``b_path`` points to the newly added file.
197 * If a file is deleted, ``b_path`` is None and ``a_path`` points to the old deleted file.
198 We still include the 'a_path' in in this case, since we want to catch
199 if any files depend on the deleted file, which would be an error.
200 """
201 for diff in diffs:
202 if diff.a_path is not None:
203 yield Path(diff.a_path).resolve()
205 if diff.b_path is not None:
206 yield Path(diff.b_path).resolve()
208 def _get_preprocessed_file_locations(self, vhd_files: set[Path]) -> set[Path]:
209 """
210 Find the location of a VUnit preprocessed file, based on the path in the modules tree.
211 Not all VHDL files are included in the simulation projects (e.g. often files that depend
212 on IP cores are excluded), hence files that can not be found in any module's simulation
213 files are ignored.
214 """
215 assert (
216 self._modules is not None
217 ), "Modules must be supplied when VUnit preprocessing is enabled"
219 result = set()
220 for vhd_file in vhd_files:
221 library_name = self._get_library_name_from_path(vhd_file)
223 if library_name is not None:
224 # Ignore that '_vunit_preprocessed_path' is type 'Path | None', since we only come
225 # if it is not 'None'.
226 preprocessed_file = (
227 self._vunit_preprocessed_path # type: ignore[operator]
228 / library_name
229 / vhd_file.name
230 )
231 assert preprocessed_file.exists(), preprocessed_file
233 result.add(preprocessed_file)
235 return result
237 def _get_library_name_from_path(self, vhd_file: Path) -> Optional[str]:
238 """
239 Returns: Library name for the given file path.
240 Will return None if no library can be found.
241 """
242 # Ignore that '_modules' is type 'Path | None', since we only come
243 # if it is not 'None'.
244 for module in self._modules: # type: ignore[union-attr]
245 for module_hdl_file in module.get_simulation_files(include_ip_cores=True):
246 if module_hdl_file.path.name == vhd_file.name:
247 return module.library_name
249 print(f"Could not find library for file {vhd_file}. It will be skipped.")
250 return None
252 def _find_testbenches(self) -> list[tuple[Any, str]]:
253 """
254 Find all testbench files that are available in the VUnit project.
256 Return:
257 The VUnit ``SourceFile`` objects and library names for the files.
258 """
259 result = []
260 for source_file in self._vunit_proj.get_source_files():
261 source_file_path = Path(source_file.name)
262 assert source_file_path.exists(), source_file_path
264 # The file is considered a testbench if it follows the tb naming pattern
265 if self._re_tb_filename.match(source_file_path.name) is not None:
266 result.append((source_file, source_file.library.name))
268 return result
270 def _source_file_depends_on_files(self, source_file: Any, files: set[Path]) -> bool:
271 """
272 Return True if the source_file depends on any of the files.
273 """
274 # Note that this includes the source_file itself. Is a list of SourceFile objects.
275 implementation_subset = self._vunit_proj.get_implementation_subset([source_file])
277 # Convert to a set of absolute Paths, for comparison with "files" which is of that type.
278 source_file_dependencies = {
279 Path(implementation_file.name).resolve()
280 for implementation_file in implementation_subset
281 }
283 intersection = source_file_dependencies & files
284 if not intersection:
285 return False
287 self._print_file_list(
288 f"Testbench {source_file.name} depends on the following files which have a diff",
289 intersection,
290 )
291 return True
293 @staticmethod
294 def _print_file_list(title: str, files: set[Path]) -> None:
295 if not files:
296 return
298 sorted_files = sorted(files)
300 print(f"{title}:")
301 for file_path in sorted_files:
302 print(f" {file_path}")
303 print()