Coverage for tsfpga/git_simulation_subset.py: 79%

1# -------------------------------------------------------------------------------------------------- 

2# Copyright (c) Lukas Vik. All rights reserved. 

3# 

4# This file is part of the tsfpga project. 

5# https://tsfpga.com 

6# https://gitlab.com/tsfpga/tsfpga 

7# -------------------------------------------------------------------------------------------------- 

8 

9import re 

10from pathlib import Path 

11 

12from git import Repo 

13 

14 

15class GitSimulationSubset: 

16 """ 

17 Find a subset of testbenches to simulate based on git history. 

18 """ 

19 

20 _re_tb_filename = re.compile(r"(tb_.+\.vhd)|(.+\_tb.vhd)") 

21 

22 def __init__( 

23 self, repo_root, reference_branch, vunit_proj, vunit_preprocessed_path=None, modules=None 

24 ): 

25 """ 

26 Arguments: 

27 repo_root (pathlib.Path): Root directory where git commands will be run. 

28 reference_branch (str): What git branch to compare against, when finding what files have 

29 changed. Typically "origin/master". 

30 vunit_proj: A vunit project with all source files and testbenches added. Will be used 

31 for dependency scanning. 

32 vunit_preprocessed_path (pathlib.Path): If location/check preprocessing is enabled 

33 in your VUnit project, supply the path to vunit_out/preprocessed. 

34 modules (ModuleList): A list of modules that are included in the VUnit project. Must be 

35 supplied only if preprocessing is enabled. 

36 """ 

37 self._repo_root = repo_root 

38 self._reference_branch = reference_branch 

39 self._vunit_proj = vunit_proj 

40 self._vunit_preprocessed_path = vunit_preprocessed_path 

41 self._modules = modules 

42 

43 if (vunit_preprocessed_path is not None) != (modules is not None): 

44 raise ValueError("Can not supply only one of vunit_preprocessed_path and modules") 

45 

46 def find_subset(self): 

47 """ 

48 Return all testbenches that have changes, or depend on files that have changes. 

49 

50 Return: 

51 list(tuple(str, str)): The testbench names and their corresponding library names. A list 

52 of tuples ("testbench name", "library name"). 

53 """ 

54 diff_files = self._find_diff_vhd_files() 

55 

56 if self._vunit_preprocessed_path: 

57 # If preprocessing is enabled, VUnit's dependency graph is based on the files that 

58 # are in the vunit_out/preprocessed folder, not in the file's original location. 

59 # So manipulate the paths to point there. 

60 diff_files = self._get_preprocessed_file_locations(diff_files) 

61 self._print_file_list("Resolved diff file locations to be", diff_files) 

62 

63 # Find all testbench files that are available 

64 testbenches = self._find_testbenches() 

65 

66 # Gather the testbenches that depend on any files that have diffs 

67 testbenches_to_run = [] 

68 for testbench_source_file, library_name in testbenches: 

69 if self._source_file_depends_on_files( 

70 source_file=testbench_source_file, 

71 files=diff_files, 

72 ): 

73 testbench_file_name = Path(testbench_source_file.name).stem 

74 testbenches_to_run.append((testbench_file_name, library_name)) 

75 

76 return testbenches_to_run 

77 

78 def _find_diff_vhd_files(self): 

79 repo = Repo(self._repo_root) 

80 

81 head_commit = repo.head.commit 

82 reference_commit = repo.commit(self._reference_branch) 

83 

84 # Local uncommitted changed 

85 working_tree_changes = head_commit.diff(None) 

86 

87 # Changes in the git log compared to the reference commit 

88 history_changes = head_commit.diff(reference_commit) 

89 

90 return self._iterate_vhd_file_diffs(diffs=working_tree_changes + history_changes) 

91 

92 def _iterate_vhd_file_diffs(self, diffs): 

93 """ 

94 Return the currently existing files that have been changed (added/renamed/modified) 

95 within any of the diffs commits. 

96 

97 Returns a set of Paths. 

98 """ 

99 files = set() 

100 

101 for diff in diffs: 

102 # The diff contains "a" -> "b" changes information. In case of file deletion, a_path 

103 # will be set but not b_path. Removed files are not included by this method. 

104 if diff.b_path is not None: 

105 b_path = Path(diff.b_path) 

106 

107 # A file can be changed in an early commit, but then removed/renamed in a 

108 # later commit. Include only files that are currently existing. 

109 if b_path.exists(): 

110 if b_path.name.endswith(".vhd"): 

111 files.add(b_path.resolve()) 

112 

113 self._print_file_list("Found git diff in the following files", files) 

114 return files 

115 

116 def _get_preprocessed_file_locations(self, vhd_files): 

117 """ 

118 Find the location of a VUnit preprocessed file, based on the path in the modules tree. 

119 Not all VHDL files are included in the simulation projects (e.g. often files that depend 

120 on IP cores are excluded), hence files that can not be found in any module's simulation 

121 files are ignored. 

122 """ 

123 result = set() 

124 for vhd_file in vhd_files: 

125 library_name = self._get_library_name_from_path(vhd_file) 

126 

127 if library_name is not None: 

128 preprocessed_file = self._vunit_preprocessed_path / library_name / vhd_file.name 

129 assert preprocessed_file.exists(), preprocessed_file 

130 

131 result.add(preprocessed_file) 

132 

133 return result 

134 

135 def _get_library_name_from_path(self, vhd_file): 

136 """ 

137 Returns (str): Library name for the given file path. 

138 Will return None if no library can be found. 

139 """ 

140 for module in self._modules: 

141 for module_hdl_file in module.get_simulation_files(include_ip_cores=True): 

142 if module_hdl_file.path.name == vhd_file.name: 

143 return module.library_name 

144 

145 print(f"Could not find library for file {vhd_file}. It will be skipped.") 

146 return None 

147 

148 def _find_testbenches(self): 

149 """ 

150 Find all testbench files that are available in the VUnit project. 

151 

152 Return: 

153 list(tuple(``SourceFile``, str)): The VUnit SourceFile objects and library names 

154 for the files. 

155 """ 

156 result = [] 

157 for source_file in self._vunit_proj.get_source_files(): 

158 source_file_path = Path(source_file.name) 

159 assert source_file_path.exists(), source_file_path 

160 

161 # The file is considered a testbench if it follows the tb naming pattern 

162 if re.fullmatch(self._re_tb_filename, source_file_path.name): 

163 result.append((source_file, source_file.library.name)) 

164 

165 return result 

166 

167 def _source_file_depends_on_files(self, source_file, files): 

168 """ 

169 Return True if the source_file depends on any of the files. 

170 """ 

171 # Note that this includes the source_file itself. Is a list of SourceFile objects. 

172 implementation_subset = self._vunit_proj.get_implementation_subset([source_file]) 

173 

174 # Convert to a set of absolute Paths, for comparison with "files" which is of that type. 

175 source_file_dependencies = { 

176 Path(implementation_file.name).resolve() 

177 for implementation_file in implementation_subset 

178 } 

179 

180 intersection = source_file_dependencies & files 

181 if not intersection: 

182 return False 

183 

184 self._print_file_list( 

185 f"Testbench {source_file.name} depends on the following files which have a diff", 

186 intersection, 

187 ) 

188 return True 

189 

190 @staticmethod 

191 def _print_file_list(title, files): 

192 print(f"{title}:") 

193 for file_path in files: 

194 print(f" {file_path}") 

195 print()