Coverage for tsfpga/git_simulation_subset.py: 74%
117 statements
« prev ^ index » next coverage.py v7.7.1, created at 2025-03-24 20:51 +0000
1# --------------------------------------------------------------------------------------------------
2# Copyright (c) Lukas Vik. All rights reserved.
3#
4# This file is part of the tsfpga project, a project platform for modern FPGA development.
5# https://tsfpga.com
6# https://github.com/tsfpga/tsfpga
7# --------------------------------------------------------------------------------------------------
9from __future__ import annotations
11import re
12from pathlib import Path
13from typing import TYPE_CHECKING, Any
15from git.repo import Repo
17from .hdl_file import HdlFile
19if TYPE_CHECKING:
20 from collections.abc import Iterable
22 from git.diff import DiffIndex
23 from vunit.ui import VUnit
24 from vunit.ui.source import SourceFile
26 from .module import BaseModule
27 from .module_list import ModuleList
29VHDL_FILE_ENDINGS = HdlFile.file_endings_mapping[HdlFile.Type.VHDL]
32class GitSimulationSubset:
33 """
34 Find a subset of testbenches to simulate based on git history.
35 """
37 # Should work with
38 # * tb_<name>.vhd
39 # * <name>_tb.vhd
40 # and possibly .vhdl as file extension.
41 _re_tb_filename = re.compile(r"^(tb_.+\.vhdl?)|(.+\_tb.vhdl?)$")
42 # Should work with .toml, .json, .yaml, etc.
43 _re_register_data_filename = re.compile(r"^regs_(.+)\.[a-z]+$")
45 def __init__(
46 self,
47 repo_root: Path,
48 reference_branch: str,
49 vunit_proj: VUnit,
50 modules: ModuleList | None = None,
51 vunit_preprocessed_path: Path | None = None,
52 ) -> None:
53 """
54 Arguments:
55 repo_root: Root directory where git commands will be run.
56 reference_branch: What git branch to compare against, when finding what files have
57 changed. Typically "origin/main" or "origin/master".
58 vunit_proj: A vunit project with all source files and testbenches added. Will be used
59 for dependency scanning.
60 modules: A list of modules that are included in the VUnit project.
62 When this argument is provided, this class will look for changes in the modules'
63 register data files, and simulate the testbenches that depend on register artifacts
64 in case of any changes.
66 This argument **must** be supplied if VUnit preprocessing is enabled.
67 vunit_preprocessed_path: If location/check preprocessing is enabled
68 in your VUnit project, supply the path to ``vunit_out/preprocessed``.
69 """
70 self._repo_root = repo_root
71 self._reference_branch = reference_branch
72 self._vunit_proj = vunit_proj
73 self._modules = modules
74 self._vunit_preprocessed_path = vunit_preprocessed_path
76 def find_subset(self) -> list[tuple[str, str]]:
77 """
78 Return all testbenches that have changes, or depend on files that have changes.
80 Return:
81 The testbench names and their corresponding library names.
82 A list of tuples ("testbench name", "library name").
83 """
84 diff_files = self._find_diff_vhd_files()
86 if self._vunit_preprocessed_path:
87 # If preprocessing is enabled, VUnit's dependency graph is based on the files that
88 # are in the vunit_out/preprocessed folder, not in the file's original location.
89 # So manipulate the paths to point there.
90 diff_files = self._get_preprocessed_file_locations(diff_files)
91 self._print_file_list("Resolved diff file locations to be", diff_files)
93 # Find all testbench files that are available
94 testbenches = self._find_testbenches()
96 # Gather the testbenches that depend on any files that have diffs
97 testbenches_to_run = []
98 for testbench_source_file in testbenches:
99 if self._source_file_depends_on_files(
100 source_file=testbench_source_file,
101 files=diff_files,
102 ):
103 testbench_file_name = Path(testbench_source_file.name).stem
104 testbenches_to_run.append((testbench_file_name, testbench_source_file.library.name))
106 return testbenches_to_run
108 def _find_diff_vhd_files(self) -> set[Path]:
109 repo = Repo(self._repo_root)
111 head_commit = repo.head.commit
112 reference_commit = repo.commit(self._reference_branch)
114 # Local uncommitted changed
115 working_tree_changes = head_commit.diff(None)
117 # Changes in the git log compared to the reference commit
118 history_changes = head_commit.diff(reference_commit)
120 all_changes: DiffIndex[Any] = working_tree_changes + history_changes
122 return self._get_vhd_files(diffs=all_changes)
124 def _get_vhd_files(self, diffs: DiffIndex[Any]) -> set[Path]:
125 """
126 Return VHDL files that have been changed (added/renamed/modified/deleted) within any
127 of the ``diffs`` commits.
129 Will also try to find VHDL files that depend on generated register artifacts that
130 have changed.
131 """
132 files = set()
134 def add_register_artifacts_if_match(
135 diff_path: Path, module_register_data_file: Path, module: BaseModule
136 ) -> None:
137 """
138 Note that Path.__eq__ does not do normalization of paths.
139 If one argument is a relative path and the other is an absolute path, they will not
140 be considered equal.
141 Hence, it is important that both paths are resolved before comparison.
142 """
143 if diff_path != module_register_data_file:
144 return
146 re_match = self._re_register_data_filename.match(module_register_data_file.name)
147 if re_match is None:
148 raise ValueError("Register data file does not use the expected naming convention")
150 register_list_name = re_match.group(1)
151 regs_pkg_path = module.register_synthesis_folder / f"{register_list_name}_regs_pkg.vhd"
153 # It is okay to add only the base register package, since all other
154 # register artifacts depend on it.
155 # This file will typically not exist yet in a CI flow, so it doesn't make sense to
156 # assert for its existence.
157 files.add(regs_pkg_path)
159 for diff_path in self._iterate_diff_paths(diffs=diffs):
160 if diff_path.name.endswith(VHDL_FILE_ENDINGS):
161 files.add(diff_path)
163 elif self._modules is not None:
164 for module in self._modules:
165 module_register_data_file = module.register_data_file
167 if isinstance(module_register_data_file, list):
168 # In case users implement a sub-class of BaseModule that has multiple
169 # register lists.
170 # This is not a standard use case that we recommend or support in general,
171 # but we support it here for convenience.
172 for data_file in module_register_data_file:
173 add_register_artifacts_if_match(
174 diff_path=diff_path,
175 module_register_data_file=data_file,
176 module=module,
177 )
179 else:
180 add_register_artifacts_if_match(
181 diff_path=diff_path,
182 module_register_data_file=module_register_data_file,
183 module=module,
184 )
186 self._print_file_list("Found git diff related to the following files", files)
187 return files
189 def _iterate_diff_paths(self, diffs: DiffIndex[Any]) -> Iterable[Path]:
190 """
191 * If a file is modified, ``a_path`` and ``b_path`` are set and point to the same file.
192 * If a file is added, ``a_path`` is None and ``b_path`` points to the newly added file.
193 * If a file is deleted, ``b_path`` is None and ``a_path`` points to the old deleted file.
194 We still include the 'a_path' in in this case, since we want to catch
195 if any files depend on the deleted file, which would be an error.
196 """
197 for diff in diffs:
198 if diff.a_path is not None:
199 yield Path(diff.a_path).resolve()
201 if diff.b_path is not None:
202 yield Path(diff.b_path).resolve()
204 def _get_preprocessed_file_locations(self, vhd_files: set[Path]) -> set[Path]:
205 """
206 Find the location of a VUnit preprocessed file, based on the path in the modules tree.
207 Not all VHDL files are included in the simulation projects (e.g. often files that depend
208 on IP cores are excluded), hence files that can not be found in any module's simulation
209 files are ignored.
210 """
211 if self._modules is None:
212 raise ValueError("Modules must be supplied when VUnit preprocessing is enabled")
214 result = set()
215 for vhd_file in vhd_files:
216 library_name = self._get_library_name_from_path(vhd_file)
218 if library_name is not None:
219 preprocessed_file = self._vunit_preprocessed_path / library_name / vhd_file.name
220 if not preprocessed_file.exists():
221 raise FileNotFoundError("Could not find file:", preprocessed_file)
223 result.add(preprocessed_file)
225 return result
227 def _get_library_name_from_path(self, vhd_file: Path) -> str | None:
228 """
229 Returns: Library name for the given file path.
230 Will return None if no library can be found.
231 """
232 # Ignore that '_modules' is type 'Path | None', since we only come
233 # if it is not 'None'.
234 for module in self._modules:
235 for module_hdl_file in module.get_simulation_files(include_ip_cores=True):
236 if module_hdl_file.path.name == vhd_file.name:
237 return module.library_name
239 print(f"Could not find library for file {vhd_file}. It will be skipped.")
240 return None
242 def _find_testbenches(self) -> list[SourceFile]:
243 """
244 Find all testbench files that are available in the VUnit project.
245 """
246 result = []
247 for source_file in self._vunit_proj.get_source_files():
248 source_file_path = Path(source_file.name)
249 if not source_file_path.exists():
250 raise FileNotFoundError("Could not find file:", source_file_path)
252 # The file is considered a testbench if it follows the tb naming pattern
253 if self._re_tb_filename.match(source_file_path.name) is not None:
254 result.append(source_file)
256 return result
258 def _source_file_depends_on_files(self, source_file: SourceFile, files: set[Path]) -> bool:
259 """
260 Return True if the source file depends on any of the files.
261 """
262 # Note that this includes the source_file itself.
263 implementation_subset = self._vunit_proj.get_implementation_subset([source_file])
265 # Convert to a set of absolute Paths, for comparison with "files" which is of that type.
266 source_file_dependencies = {
267 Path(implementation_file.name).resolve()
268 for implementation_file in implementation_subset
269 }
271 intersection = source_file_dependencies & files
272 if not intersection:
273 return False
275 self._print_file_list(
276 f"Testbench {source_file.name} depends on the following files which have a diff",
277 intersection,
278 )
279 return True
281 @staticmethod
282 def _print_file_list(title: str, files: set[Path]) -> None:
283 if not files:
284 return
286 sorted_files = sorted(files)
288 print(f"{title}:")
289 for file_path in sorted_files:
290 print(f" {file_path}")
291 print()