Coverage for tsfpga/git_simulation_subset.py: 74%

117 statements  

« prev     ^ index     » next       coverage.py v7.8.0, created at 2025-04-01 20:51 +0000

1# -------------------------------------------------------------------------------------------------- 

2# Copyright (c) Lukas Vik. All rights reserved. 

3# 

4# This file is part of the tsfpga project, a project platform for modern FPGA development. 

5# https://tsfpga.com 

6# https://github.com/tsfpga/tsfpga 

7# -------------------------------------------------------------------------------------------------- 

8 

9from __future__ import annotations 

10 

11import re 

12from pathlib import Path 

13from typing import TYPE_CHECKING, Any 

14 

15from git.repo import Repo 

16 

17from .hdl_file import HdlFile 

18 

19if TYPE_CHECKING: 

20 from collections.abc import Iterable 

21 

22 from git.diff import DiffIndex 

23 from vunit.ui import VUnit 

24 from vunit.ui.source import SourceFile 

25 

26 from .module import BaseModule 

27 from .module_list import ModuleList 

28 

29VHDL_FILE_ENDINGS = HdlFile.file_endings_mapping[HdlFile.Type.VHDL] 

30 

31 

32class GitSimulationSubset: 

33 """ 

34 Find a subset of testbenches to simulate based on git history. 

35 """ 

36 

37 # Should work with 

38 # * tb_<name>.vhd 

39 # * <name>_tb.vhd 

40 # and possibly .vhdl as file extension. 

41 _re_tb_filename = re.compile(r"^(tb_.+\.vhdl?)|(.+\_tb.vhdl?)$") 

42 # Should work with .toml, .json, .yaml, etc. 

43 _re_register_data_filename = re.compile(r"^regs_(.+)\.[a-z]+$") 

44 

45 def __init__( 

46 self, 

47 repo_root: Path, 

48 reference_branch: str, 

49 vunit_proj: VUnit, 

50 modules: ModuleList | None = None, 

51 vunit_preprocessed_path: Path | None = None, 

52 ) -> None: 

53 """ 

54 Arguments: 

55 repo_root: Root directory where git commands will be run. 

56 reference_branch: What git branch to compare against, when finding what files have 

57 changed. Typically "origin/main" or "origin/master". 

58 vunit_proj: A vunit project with all source files and testbenches added. Will be used 

59 for dependency scanning. 

60 modules: A list of modules that are included in the VUnit project. 

61 

62 When this argument is provided, this class will look for changes in the modules' 

63 register data files, and simulate the testbenches that depend on register artifacts 

64 in case of any changes. 

65 

66 This argument **must** be supplied if VUnit preprocessing is enabled. 

67 vunit_preprocessed_path: If location/check preprocessing is enabled 

68 in your VUnit project, supply the path to ``vunit_out/preprocessed``. 

69 """ 

70 self._repo_root = repo_root 

71 self._reference_branch = reference_branch 

72 self._vunit_proj = vunit_proj 

73 self._modules = modules 

74 self._vunit_preprocessed_path = vunit_preprocessed_path 

75 

76 def find_subset(self) -> list[tuple[str, str]]: 

77 """ 

78 Return all testbenches that have changes, or depend on files that have changes. 

79 

80 Return: 

81 The testbench names and their corresponding library names. 

82 A list of tuples ("testbench name", "library name"). 

83 """ 

84 diff_files = self._find_diff_vhd_files() 

85 

86 if self._vunit_preprocessed_path: 

87 # If preprocessing is enabled, VUnit's dependency graph is based on the files that 

88 # are in the vunit_out/preprocessed folder, not in the file's original location. 

89 # So manipulate the paths to point there. 

90 diff_files = self._get_preprocessed_file_locations(diff_files) 

91 self._print_file_list("Resolved diff file locations to be", diff_files) 

92 

93 # Find all testbench files that are available 

94 testbenches = self._find_testbenches() 

95 

96 # Gather the testbenches that depend on any files that have diffs 

97 testbenches_to_run = [] 

98 for testbench_source_file in testbenches: 

99 if self._source_file_depends_on_files( 

100 source_file=testbench_source_file, 

101 files=diff_files, 

102 ): 

103 testbench_file_name = Path(testbench_source_file.name).stem 

104 testbenches_to_run.append((testbench_file_name, testbench_source_file.library.name)) 

105 

106 return testbenches_to_run 

107 

108 def _find_diff_vhd_files(self) -> set[Path]: 

109 repo = Repo(self._repo_root) 

110 

111 head_commit = repo.head.commit 

112 reference_commit = repo.commit(self._reference_branch) 

113 

114 # Local uncommitted changed 

115 working_tree_changes = head_commit.diff(None) 

116 

117 # Changes in the git log compared to the reference commit 

118 history_changes = head_commit.diff(reference_commit) 

119 

120 all_changes: DiffIndex[Any] = working_tree_changes + history_changes 

121 

122 return self._get_vhd_files(diffs=all_changes) 

123 

124 def _get_vhd_files(self, diffs: DiffIndex[Any]) -> set[Path]: 

125 """ 

126 Return VHDL files that have been changed (added/renamed/modified/deleted) within any 

127 of the ``diffs`` commits. 

128 

129 Will also try to find VHDL files that depend on generated register artifacts that 

130 have changed. 

131 """ 

132 files = set() 

133 

134 def add_register_artifacts_if_match( 

135 diff_path: Path, module_register_data_file: Path, module: BaseModule 

136 ) -> None: 

137 """ 

138 Note that Path.__eq__ does not do normalization of paths. 

139 If one argument is a relative path and the other is an absolute path, they will not 

140 be considered equal. 

141 Hence, it is important that both paths are resolved before comparison. 

142 """ 

143 if diff_path != module_register_data_file: 

144 return 

145 

146 re_match = self._re_register_data_filename.match(module_register_data_file.name) 

147 if re_match is None: 

148 raise ValueError("Register data file does not use the expected naming convention") 

149 

150 register_list_name = re_match.group(1) 

151 regs_pkg_path = module.register_synthesis_folder / f"{register_list_name}_regs_pkg.vhd" 

152 

153 # It is okay to add only the base register package, since all other 

154 # register artifacts depend on it. 

155 # This file will typically not exist yet in a CI flow, so it doesn't make sense to 

156 # assert for its existence. 

157 files.add(regs_pkg_path) 

158 

159 for diff_path in self._iterate_diff_paths(diffs=diffs): 

160 if diff_path.name.endswith(VHDL_FILE_ENDINGS): 

161 files.add(diff_path) 

162 

163 elif self._modules is not None: 

164 for module in self._modules: 

165 module_register_data_file = module.register_data_file 

166 

167 if isinstance(module_register_data_file, list): 

168 # In case users implement a sub-class of BaseModule that has multiple 

169 # register lists. 

170 # This is not a standard use case that we recommend or support in general, 

171 # but we support it here for convenience. 

172 for data_file in module_register_data_file: 

173 add_register_artifacts_if_match( 

174 diff_path=diff_path, 

175 module_register_data_file=data_file, 

176 module=module, 

177 ) 

178 

179 else: 

180 add_register_artifacts_if_match( 

181 diff_path=diff_path, 

182 module_register_data_file=module_register_data_file, 

183 module=module, 

184 ) 

185 

186 self._print_file_list("Found git diff related to the following files", files) 

187 return files 

188 

189 def _iterate_diff_paths(self, diffs: DiffIndex[Any]) -> Iterable[Path]: 

190 """ 

191 * If a file is modified, ``a_path`` and ``b_path`` are set and point to the same file. 

192 * If a file is added, ``a_path`` is None and ``b_path`` points to the newly added file. 

193 * If a file is deleted, ``b_path`` is None and ``a_path`` points to the old deleted file. 

194 We still include the 'a_path' in in this case, since we want to catch 

195 if any files depend on the deleted file, which would be an error. 

196 """ 

197 for diff in diffs: 

198 if diff.a_path is not None: 

199 yield Path(diff.a_path).resolve() 

200 

201 if diff.b_path is not None: 

202 yield Path(diff.b_path).resolve() 

203 

204 def _get_preprocessed_file_locations(self, vhd_files: set[Path]) -> set[Path]: 

205 """ 

206 Find the location of a VUnit preprocessed file, based on the path in the modules tree. 

207 Not all VHDL files are included in the simulation projects (e.g. often files that depend 

208 on IP cores are excluded), hence files that can not be found in any module's simulation 

209 files are ignored. 

210 """ 

211 if self._modules is None: 

212 raise ValueError("Modules must be supplied when VUnit preprocessing is enabled") 

213 

214 result = set() 

215 for vhd_file in vhd_files: 

216 library_name = self._get_library_name_from_path(vhd_file) 

217 

218 if library_name is not None: 

219 preprocessed_file = self._vunit_preprocessed_path / library_name / vhd_file.name 

220 if not preprocessed_file.exists(): 

221 raise FileNotFoundError("Could not find file:", preprocessed_file) 

222 

223 result.add(preprocessed_file) 

224 

225 return result 

226 

227 def _get_library_name_from_path(self, vhd_file: Path) -> str | None: 

228 """ 

229 Returns: Library name for the given file path. 

230 Will return None if no library can be found. 

231 """ 

232 # Ignore that '_modules' is type 'Path | None', since we only come 

233 # if it is not 'None'. 

234 for module in self._modules: 

235 for module_hdl_file in module.get_simulation_files(include_ip_cores=True): 

236 if module_hdl_file.path.name == vhd_file.name: 

237 return module.library_name 

238 

239 print(f"Could not find library for file {vhd_file}. It will be skipped.") 

240 return None 

241 

242 def _find_testbenches(self) -> list[SourceFile]: 

243 """ 

244 Find all testbench files that are available in the VUnit project. 

245 """ 

246 result = [] 

247 for source_file in self._vunit_proj.get_source_files(): 

248 source_file_path = Path(source_file.name) 

249 if not source_file_path.exists(): 

250 raise FileNotFoundError("Could not find file:", source_file_path) 

251 

252 # The file is considered a testbench if it follows the tb naming pattern 

253 if self._re_tb_filename.match(source_file_path.name) is not None: 

254 result.append(source_file) 

255 

256 return result 

257 

258 def _source_file_depends_on_files(self, source_file: SourceFile, files: set[Path]) -> bool: 

259 """ 

260 Return True if the source file depends on any of the files. 

261 """ 

262 # Note that this includes the source_file itself. 

263 implementation_subset = self._vunit_proj.get_implementation_subset([source_file]) 

264 

265 # Convert to a set of absolute Paths, for comparison with "files" which is of that type. 

266 source_file_dependencies = { 

267 Path(implementation_file.name).resolve() 

268 for implementation_file in implementation_subset 

269 } 

270 

271 intersection = source_file_dependencies & files 

272 if not intersection: 

273 return False 

274 

275 self._print_file_list( 

276 f"Testbench {source_file.name} depends on the following files which have a diff", 

277 intersection, 

278 ) 

279 return True 

280 

281 @staticmethod 

282 def _print_file_list(title: str, files: set[Path]) -> None: 

283 if not files: 

284 return 

285 

286 sorted_files = sorted(files) 

287 

288 print(f"{title}:") 

289 for file_path in sorted_files: 

290 print(f" {file_path}") 

291 print()