Coverage for tsfpga/git_simulation_subset.py: 79%
106 statements
« prev ^ index » next coverage.py v7.6.7, created at 2024-11-20 20:51 +0000
1# --------------------------------------------------------------------------------------------------
2# Copyright (c) Lukas Vik. All rights reserved.
3#
4# This file is part of the tsfpga project, a project platform for modern FPGA development.
5# https://tsfpga.com
6# https://github.com/tsfpga/tsfpga
7# --------------------------------------------------------------------------------------------------
9# Standard libraries
10import re
11from collections.abc import Iterable
12from pathlib import Path
13from typing import TYPE_CHECKING, Any, Optional
15# Third party libraries
16from git.repo import Repo
18# Local folder libraries
19from .hdl_file import HdlFile
21if TYPE_CHECKING:
22 # Third party libraries
23 from vunit.ui import VUnit
25 # Local folder libraries
26 from .module import BaseModule
27 from .module_list import ModuleList
29VHDL_FILE_ENDINGS = HdlFile.file_endings_mapping[HdlFile.Type.VHDL]
32class GitSimulationSubset:
33 """
34 Find a subset of testbenches to simulate based on git history.
35 """
37 # Should work with
38 # * tb_<name>.vhd
39 # * <name>_tb.vhd
40 # and possibly .vhdl as file extension.
41 _re_tb_filename = re.compile(r"^(tb_.+\.vhdl?)|(.+\_tb.vhdl?)$")
42 # Should work with .toml, .json, .yaml, etc.
43 _re_register_data_filename = re.compile(r"^regs_(.+)\.[a-z]+$")
45 def __init__(
46 self,
47 repo_root: Path,
48 reference_branch: str,
49 vunit_proj: "VUnit",
50 modules: Optional["ModuleList"] = None,
51 vunit_preprocessed_path: Optional[Path] = None,
52 ) -> None:
53 """
54 Arguments:
55 repo_root: Root directory where git commands will be run.
56 reference_branch: What git branch to compare against, when finding what files have
57 changed. Typically "origin/main" or "origin/master".
58 vunit_proj: A vunit project with all source files and testbenches added. Will be used
59 for dependency scanning.
60 modules: A list of modules that are included in the VUnit project.
62 When this argument is provided, this class will look for changes in the modules'
63 register data files, and simulate the testbenches that depend on register artifacts
64 in case of any changes.
66 This argument **must** be supplied if VUnit preprocessing is enabled.
67 vunit_preprocessed_path: If location/check preprocessing is enabled
68 in your VUnit project, supply the path to ``vunit_out/preprocessed``.
69 """
70 self._repo_root = repo_root
71 self._reference_branch = reference_branch
72 self._vunit_proj = vunit_proj
73 self._modules = modules
74 self._vunit_preprocessed_path = vunit_preprocessed_path
76 def find_subset(self) -> list[tuple[str, str]]:
77 """
78 Return all testbenches that have changes, or depend on files that have changes.
80 Return:
81 The testbench names and their corresponding library names.
82 A list of tuples ("testbench name", "library name").
83 """
84 diff_files = self._find_diff_vhd_files()
86 if self._vunit_preprocessed_path:
87 # If preprocessing is enabled, VUnit's dependency graph is based on the files that
88 # are in the vunit_out/preprocessed folder, not in the file's original location.
89 # So manipulate the paths to point there.
90 diff_files = self._get_preprocessed_file_locations(diff_files)
91 self._print_file_list("Resolved diff file locations to be", diff_files)
93 # Find all testbench files that are available
94 testbenches = self._find_testbenches()
96 # Gather the testbenches that depend on any files that have diffs
97 testbenches_to_run = []
98 for testbench_source_file, library_name in testbenches:
99 if self._source_file_depends_on_files(
100 source_file=testbench_source_file,
101 files=diff_files,
102 ):
103 testbench_file_name = Path(testbench_source_file.name).stem
104 testbenches_to_run.append((testbench_file_name, library_name))
106 return testbenches_to_run
108 def _find_diff_vhd_files(self) -> set[Path]:
109 repo = Repo(self._repo_root)
111 head_commit = repo.head.commit
112 reference_commit = repo.commit(self._reference_branch)
114 # Local uncommitted changed
115 working_tree_changes = head_commit.diff(None)
117 # Changes in the git log compared to the reference commit
118 history_changes = head_commit.diff(reference_commit)
120 return self._iterate_vhd_file_diffs(diffs=working_tree_changes + history_changes)
122 def _iterate_vhd_file_diffs(self, diffs: Any) -> set[Path]:
123 """
124 Return the currently existing VHDL files that have been changed (added/renamed/modified)
125 within any of the ``diffs`` commits.
127 Will also try to find VHDL files that depend on generated register artifacts that
128 have changed.
129 """
130 files = set()
132 def add_register_artifacts_if_match(
133 diff_path: Path, module_register_data_file: Path, module: "BaseModule"
134 ) -> None:
135 """
136 Note that Path.__eq__ does not do normalization of paths.
137 If one argument is a relative path and the other is an absolute path, they will not
138 be considered equal.
139 Hence, it is important that both paths are resolved before comparison.
140 """
141 if diff_path != module_register_data_file:
142 return
144 re_match = self._re_register_data_filename.match(module_register_data_file.name)
145 assert (
146 re_match is not None
147 ), "Register data file does not use the expected naming convention"
149 register_list_name = re_match.group(1)
150 regs_pkg_path = module.register_synthesis_folder / f"{register_list_name}_regs_pkg.vhd"
152 # It is okay to add only the base register package, since all other
153 # register artifacts depend on it.
154 # This file will typically not exist yet in a CI flow, so it doesn't make sense to
155 # assert for its existence.
156 files.add(regs_pkg_path)
158 for diff_path in self._iterate_diff_paths(diffs=diffs):
159 if diff_path.name.endswith(VHDL_FILE_ENDINGS):
160 files.add(diff_path)
162 elif self._modules is not None:
163 for module in self._modules:
164 module_register_data_file = module.register_data_file
166 if isinstance(module_register_data_file, list):
167 # In users implement a sub-class of BaseModule that has multiple register
168 # lists. This is not a standard use case, but we support it here.
169 for data_file in module_register_data_file:
170 add_register_artifacts_if_match(
171 diff_path=diff_path,
172 module_register_data_file=data_file,
173 module=module,
174 )
176 else:
177 add_register_artifacts_if_match(
178 diff_path=diff_path,
179 module_register_data_file=module_register_data_file,
180 module=module,
181 )
183 self._print_file_list("Found git diff in the following files", files)
184 return files
186 def _iterate_diff_paths(self, diffs: Any) -> Iterable[Path]:
187 for diff in diffs:
188 # The diff contains "a" -> "b" changes information. In case of file deletion, a_path
189 # will be set but not b_path. Removed files are not included by this method.
190 if diff.b_path is not None:
191 b_path = Path(diff.b_path)
193 # A file can be changed in an early commit, but then removed/renamed in a
194 # later commit. Include only files that are currently existing.
195 if b_path.exists():
196 yield b_path.resolve()
198 def _get_preprocessed_file_locations(self, vhd_files: set[Path]) -> set[Path]:
199 """
200 Find the location of a VUnit preprocessed file, based on the path in the modules tree.
201 Not all VHDL files are included in the simulation projects (e.g. often files that depend
202 on IP cores are excluded), hence files that can not be found in any module's simulation
203 files are ignored.
204 """
205 assert (
206 self._modules is not None
207 ), "Modules must be supplied when VUnit preprocessing is enabled"
209 result = set()
210 for vhd_file in vhd_files:
211 library_name = self._get_library_name_from_path(vhd_file)
213 if library_name is not None:
214 # Ignore that '_vunit_preprocessed_path' is type 'Path | None', since we only come
215 # if it is not 'None'.
216 preprocessed_file = (
217 self._vunit_preprocessed_path # type: ignore[operator]
218 / library_name
219 / vhd_file.name
220 )
221 assert preprocessed_file.exists(), preprocessed_file
223 result.add(preprocessed_file)
225 return result
227 def _get_library_name_from_path(self, vhd_file: Path) -> Optional[str]:
228 """
229 Returns: Library name for the given file path.
230 Will return None if no library can be found.
231 """
232 # Ignore that '_modules' is type 'Path | None', since we only come
233 # if it is not 'None'.
234 for module in self._modules: # type: ignore[union-attr]
235 for module_hdl_file in module.get_simulation_files(include_ip_cores=True):
236 if module_hdl_file.path.name == vhd_file.name:
237 return module.library_name
239 print(f"Could not find library for file {vhd_file}. It will be skipped.")
240 return None
242 def _find_testbenches(self) -> list[tuple[Any, str]]:
243 """
244 Find all testbench files that are available in the VUnit project.
246 Return:
247 The VUnit ``SourceFile`` objects and library names for the files.
248 """
249 result = []
250 for source_file in self._vunit_proj.get_source_files():
251 source_file_path = Path(source_file.name)
252 assert source_file_path.exists(), source_file_path
254 # The file is considered a testbench if it follows the tb naming pattern
255 if self._re_tb_filename.match(source_file_path.name) is not None:
256 result.append((source_file, source_file.library.name))
258 return result
260 def _source_file_depends_on_files(self, source_file: Any, files: set[Path]) -> bool:
261 """
262 Return True if the source_file depends on any of the files.
263 """
264 # Note that this includes the source_file itself. Is a list of SourceFile objects.
265 implementation_subset = self._vunit_proj.get_implementation_subset([source_file])
267 # Convert to a set of absolute Paths, for comparison with "files" which is of that type.
268 source_file_dependencies = {
269 Path(implementation_file.name).resolve()
270 for implementation_file in implementation_subset
271 }
273 intersection = source_file_dependencies & files
274 if not intersection:
275 return False
277 self._print_file_list(
278 f"Testbench {source_file.name} depends on the following files which have a diff",
279 intersection,
280 )
281 return True
283 @staticmethod
284 def _print_file_list(title: str, files: set[Path]) -> None:
285 print(f"{title}:")
286 for file_path in files:
287 print(f" {file_path}")
288 print()