Coverage for tsfpga/git_simulation_subset.py: 79%

80 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-29 20:01 +0000

1# -------------------------------------------------------------------------------------------------- 

2# Copyright (c) Lukas Vik. All rights reserved. 

3# 

4# This file is part of the tsfpga project, a project platform for modern FPGA development. 

5# https://tsfpga.com 

6# https://gitlab.com/tsfpga/tsfpga 

7# -------------------------------------------------------------------------------------------------- 

8 

9# Standard libraries 

10import re 

11from pathlib import Path 

12 

13# Third party libraries 

14from git import Repo 

15 

16 

17class GitSimulationSubset: 

18 """ 

19 Find a subset of testbenches to simulate based on git history. 

20 """ 

21 

22 _re_tb_filename = re.compile(r"(tb_.+\.vhd)|(.+\_tb.vhd)") 

23 

24 def __init__( 

25 self, repo_root, reference_branch, vunit_proj, vunit_preprocessed_path=None, modules=None 

26 ): 

27 """ 

28 Arguments: 

29 repo_root (pathlib.Path): Root directory where git commands will be run. 

30 reference_branch (str): What git branch to compare against, when finding what files have 

31 changed. Typically "origin/main" or "origin/master". 

32 vunit_proj: A vunit project with all source files and testbenches added. Will be used 

33 for dependency scanning. 

34 vunit_preprocessed_path (pathlib.Path): If location/check preprocessing is enabled 

35 in your VUnit project, supply the path to vunit_out/preprocessed. 

36 modules (ModuleList): A list of modules that are included in the VUnit project. Must be 

37 supplied only if preprocessing is enabled. 

38 """ 

39 self._repo_root = repo_root 

40 self._reference_branch = reference_branch 

41 self._vunit_proj = vunit_proj 

42 self._vunit_preprocessed_path = vunit_preprocessed_path 

43 self._modules = modules 

44 

45 if (vunit_preprocessed_path is not None) != (modules is not None): 

46 raise ValueError("Can not supply only one of vunit_preprocessed_path and modules") 

47 

48 def find_subset(self): 

49 """ 

50 Return all testbenches that have changes, or depend on files that have changes. 

51 

52 Return: 

53 list(tuple(str, str)): The testbench names and their corresponding library names. A list 

54 of tuples ("testbench name", "library name"). 

55 """ 

56 diff_files = self._find_diff_vhd_files() 

57 

58 if self._vunit_preprocessed_path: 

59 # If preprocessing is enabled, VUnit's dependency graph is based on the files that 

60 # are in the vunit_out/preprocessed folder, not in the file's original location. 

61 # So manipulate the paths to point there. 

62 diff_files = self._get_preprocessed_file_locations(diff_files) 

63 self._print_file_list("Resolved diff file locations to be", diff_files) 

64 

65 # Find all testbench files that are available 

66 testbenches = self._find_testbenches() 

67 

68 # Gather the testbenches that depend on any files that have diffs 

69 testbenches_to_run = [] 

70 for testbench_source_file, library_name in testbenches: 

71 if self._source_file_depends_on_files( 

72 source_file=testbench_source_file, 

73 files=diff_files, 

74 ): 

75 testbench_file_name = Path(testbench_source_file.name).stem 

76 testbenches_to_run.append((testbench_file_name, library_name)) 

77 

78 return testbenches_to_run 

79 

80 def _find_diff_vhd_files(self): 

81 repo = Repo(self._repo_root) 

82 

83 head_commit = repo.head.commit 

84 reference_commit = repo.commit(self._reference_branch) 

85 

86 # Local uncommitted changed 

87 working_tree_changes = head_commit.diff(None) 

88 

89 # Changes in the git log compared to the reference commit 

90 history_changes = head_commit.diff(reference_commit) 

91 

92 return self._iterate_vhd_file_diffs(diffs=working_tree_changes + history_changes) 

93 

94 def _iterate_vhd_file_diffs(self, diffs): 

95 """ 

96 Return the currently existing files that have been changed (added/renamed/modified) 

97 within any of the diffs commits. 

98 

99 Returns a set of Paths. 

100 """ 

101 files = set() 

102 

103 for diff in diffs: 

104 # The diff contains "a" -> "b" changes information. In case of file deletion, a_path 

105 # will be set but not b_path. Removed files are not included by this method. 

106 if diff.b_path is not None: 

107 b_path = Path(diff.b_path) 

108 

109 # A file can be changed in an early commit, but then removed/renamed in a 

110 # later commit. Include only files that are currently existing. 

111 if b_path.exists(): 

112 if b_path.name.endswith(".vhd"): 

113 files.add(b_path.resolve()) 

114 

115 self._print_file_list("Found git diff in the following files", files) 

116 return files 

117 

118 def _get_preprocessed_file_locations(self, vhd_files): 

119 """ 

120 Find the location of a VUnit preprocessed file, based on the path in the modules tree. 

121 Not all VHDL files are included in the simulation projects (e.g. often files that depend 

122 on IP cores are excluded), hence files that can not be found in any module's simulation 

123 files are ignored. 

124 """ 

125 result = set() 

126 for vhd_file in vhd_files: 

127 library_name = self._get_library_name_from_path(vhd_file) 

128 

129 if library_name is not None: 

130 preprocessed_file = self._vunit_preprocessed_path / library_name / vhd_file.name 

131 assert preprocessed_file.exists(), preprocessed_file 

132 

133 result.add(preprocessed_file) 

134 

135 return result 

136 

137 def _get_library_name_from_path(self, vhd_file): 

138 """ 

139 Returns (str): Library name for the given file path. 

140 Will return None if no library can be found. 

141 """ 

142 for module in self._modules: 

143 for module_hdl_file in module.get_simulation_files(include_ip_cores=True): 

144 if module_hdl_file.path.name == vhd_file.name: 

145 return module.library_name 

146 

147 print(f"Could not find library for file {vhd_file}. It will be skipped.") 

148 return None 

149 

150 def _find_testbenches(self): 

151 """ 

152 Find all testbench files that are available in the VUnit project. 

153 

154 Return: 

155 list(tuple(``SourceFile``, str)): The VUnit SourceFile objects and library names 

156 for the files. 

157 """ 

158 result = [] 

159 for source_file in self._vunit_proj.get_source_files(): 

160 source_file_path = Path(source_file.name) 

161 assert source_file_path.exists(), source_file_path 

162 

163 # The file is considered a testbench if it follows the tb naming pattern 

164 if re.fullmatch(self._re_tb_filename, source_file_path.name): 

165 result.append((source_file, source_file.library.name)) 

166 

167 return result 

168 

169 def _source_file_depends_on_files(self, source_file, files): 

170 """ 

171 Return True if the source_file depends on any of the files. 

172 """ 

173 # Note that this includes the source_file itself. Is a list of SourceFile objects. 

174 implementation_subset = self._vunit_proj.get_implementation_subset([source_file]) 

175 

176 # Convert to a set of absolute Paths, for comparison with "files" which is of that type. 

177 source_file_dependencies = { 

178 Path(implementation_file.name).resolve() 

179 for implementation_file in implementation_subset 

180 } 

181 

182 intersection = source_file_dependencies & files 

183 if not intersection: 

184 return False 

185 

186 self._print_file_list( 

187 f"Testbench {source_file.name} depends on the following files which have a diff", 

188 intersection, 

189 ) 

190 return True 

191 

192 @staticmethod 

193 def _print_file_list(title, files): 

194 print(f"{title}:") 

195 for file_path in files: 

196 print(f" {file_path}") 

197 print()