1.思路
1）获取全量代码覆盖率报告；
2）指定两个版本对比，得到增量代码；
3）通过增量代码获取到增量包名、类、方法、新增行数组成的字典；
4）通过全量覆盖率文件获取到文件增量代码行、增量代码行数、覆盖行、覆盖行数；
5）循环读取，更改各个目录下的index.html和类名.html文件；显示新增行数、覆盖行数和覆盖率；
6）循环读取，更改类名.java.html文件，在新增行数前加上蓝色的钻石标记；
2.实现
diff_processor.py 执行增量筛选操作
class DiffProcessor():
def __init__(self, diffpath,code_path,coco_path):
'''
:param diffpath:gitdiff代码文件路径
:param code_path: 源码路径
:param coco_path: jacoco全量覆盖率报告路径
'''
self.diffpath = diffpath
self.code_path = code_path
self.coco_path = coco_path
2.1 get_diff方法：
1）循环读取gitdiff文件，如果当前行是以"diff --git"开头的，获取完整文件名，如：src/main/webapp/WEB-INF/default/jsp/tlfund/management/fund_transfer_record_list.jsp;
2）如果当前行中包含"@@XXX @@"字样的，获取classname；
3）如果当前行是以“-”开头的，跳过；
4）过滤方法名：如果当前行不是以"+//"、“//”开头的，包含"private"或"public"和"("，且"function"、“=”、“if(”、"if ("、"for "、“for(”、“catch”、"logger."、“.”不在行内，且不是以";"结尾的，获取当前行并过滤出方法名；如果方法名不是"if"、"for"且"{"、"}"、"."、"+"、"@"不在行内的；将方法名添加到字典中，如果方法名已存在，将当前行添加到方法名的列表中；
5）如果当前行是以"+"开头的，添加到对应的方法名列表中；
6）最终返回{"文件名":{"diff_voids":{{方法名:[新增行]},'diff_lines':[所有新增行]}}的字典；
def get_diff(self):
"""获取diff详情"""
with open(self.diffpath, 'r+') as e:
data = e.read()
diff = data.split("
")
ret = {}
void_dics = {}
diff_lines = []
file_name = ""
line_void = ""
current_line = 0
for line in diff:
if line.startswith('diff --git'):
# 进入新的block
if file_name != "":
ret[file_name] = {}
ret[file_name]["diff_lines"] = diff_lines
ret[file_name]['diff_voids'] = void_dics
file_name = re.findall('b/(S+)$', line)[0]
diff_lines = []
void_dics = {}
current_line = 0
classname = ""
elif re.match('@@ -d+,d+ +(d+),d+ @@', line):
match = re.match('@@ -d+,d+ +(d+),d+ @@', line)
current_line = int(match.group(1)) - 1
if "class" in line:
classname = line.split("class")[-1].split(" ")[1]
if classname and classname not in void_dics.keys():
void_dics[classname]=[]
line_void = ""
elif classname in void_dics:
line_void = ""
elif line.startswith("-"):
continue
elif not line.startswith("+//") and not line.startswith("//") and ("public" in line or "private" in line) and "(" in line and "function" not in line and "=" not in line
and "if(" not in line and "if (" not in line and "for " not in line and "for(" not in line and "catch" not in line and "logger." not in line and not line.strip(" ").endswith(";")
and "." not in line:
current_line += 1
line_void = line.split("(")[0].split(" ")[-1].strip("//").strip(" ")
if line_void and line_void != "if" and line_void != "for" and "{" not in line_void and "}" not in line_void and "." not in line_void and "+" not in line_void and "@" not in line_void:
if line_void in void_dics.keys() and isinstance(void_dics[line_void], list):
void_dics[line_void].append(current_line)
else:
void_dics[line_void] = []
elif line.startswith("+") and not line.startswith('+++'):
current_line += 1
diff_lines.append(current_line)
if line_void in void_dics.keys() and isinstance(void_dics[line_void], list):
void_dics[line_void].append(current_line)
elif classname:
void_dics[classname].append(current_line)
else:
current_line += 1
ret[file_name] = {}
ret[file_name]["diff_lines"] = diff_lines
ret[file_name]['diff_voids'] = void_dics
return ret
2.2 is_interface方法：
根据传递过来的文件名，判断是否为接口；返回判断状态
def is_interface(self, file_name):
"""判断某个文件是否是接口"""
ret = False
name = re.search('(w+).java$', file_name).group(1)
reg_interface = re.compile('publics+interfaces+{}'.format(name))
with open(file_name) as fp:
for line in fp:
line = line.strip()
match = re.match(reg_interface, line)
if match:
ret = True
break
return ret
2.3 modify_html方法
根据传递过来的文件路径和新增总行列表，对新增行通过class新增蓝色钻石标志，并统计新增行、覆盖行、新增行数和覆盖行数后返回；
def modify_html(self, html_file_name, diff_lines):
new_line_count = 0
cover_line_count = 0
content = []
new_lines = []
cover_lines = []
with open(html_file_name, 'r') as fp:
content = fp.readlines()
for i in range(1, len(content)):
if i + 1 in diff_lines:
match = re.search('class="([^"]+)"', content[i])
if match:
content[i] = re.sub('class="([^"]+)"', lambda m: 'class="{}-diff"'.format(m.group(1)) if "diff" not in m.group(1) else "class={}".format(m.group(1)), content[i])
css_class = match.group(1)
new_line_count += 1
new_lines.append(i+1)
if css_class.startswith("fc") or css_class.startswith("pc"):
cover_line_count += 1
cover_lines.append(i+1)
with open(html_file_name, 'w') as fp:
fp.write("".join(content))
return new_line_count,cover_line_count,new_lines,cover_lines
2.4 get_package方法
通过传递过来的完整文件名，获取包名
def get_package(self, file_name):
"""获取package名"""
ret = ''
with open(file_name) as fp:
for line in fp:
line = line.strip()
match = re.match('packages+(S+);', line)
if match:
ret = match.group(1)
break
return ret
2.5 resolve_file_info方法
根据传递过来的完整文件名，组成完整路径，调用get_package方法获取包名，通过正则获取类名，调用is_interface方法判断是否为接口方法
def resolve_file_info(self,file_name):
module_name = file_name.split('/')[0]
full_path = os.path.join(self.code_path,file_name)
package = self.get_package(full_path)
class_ = re.search('(w+).java$',file_name).group(1)
is_interface = self.is_interface(full_path)
return (module_name,package, class_, is_interface)
2.6 process_diff方法
调用get_diff方法获取返回的字典；返回两个字典：ret是{"new":新增行数,"cover":覆盖行数,"new_lines":新增的行,"cover_lines":覆盖行}，diff_result是get_diff方法返回的字典；
def process_diff(self):
'''
过滤出新增包名、类和新增代码行、已覆盖行数并返回
'''
ret = {}
diff_result = self.get_diff()
for file_name in diff_result:
# 过滤掉只有删除，没有新增的代码
if diff_result[file_name]['diff_lines'] == []:
continue
# 过滤掉非 java 文件和测试代码
if file_name.endswith("$.java") or not file_name.endswith(".java") or 'src/test/java/' in file_name:
continue
module_name, package, class_, is_interface = self.resolve_file_info(file_name)
# 过滤掉接口和非指定的module
if is_interface:
continue
html_file_name = os.path.join(self.coco_path,package,"{}.java.html".format(class_))
new_line_count,cover_line_count,new_lines,cover_lines= self.modify_html(html_file_name, diff_result[file_name]['diff_lines'])
# 信息存进返回值
if package not in ret:
ret[package] = {}
ret[package][class_] = {"new": new_line_count,"cover": cover_line_count,"new_lines":new_lines,"cover_lines":cover_lines}
return ret,diff_result
2.7 Del_Dr方法
该方法需要传入的参数为：要修改的html文件的路径、html页面要保留的类名列表或文件列表、process_diff方法返回的ret字典、要修改的类型(root/package/file)；通过传入参数针对多余的文件进行删除操作并修改html页面元素展示增量报告；
def Del_Dr(self, htmlpath, dirlist, ret, filetype, *kwargs):
'''
该函数的目的是为了修改html页面的内容
:param htmlpath:要修改的html文件的路径
:param dirlist:html页面要保留的类名列表或文件
:param ret:ret
:param diff_results:diff文件过滤的字典
:param filetype:需要修改的类型(root指文件根目录，package指包目录下，file指文件类型)
'''
with open(htmlpath, 'r') as e:
html_doc = "".join(e.readlines())
soup = BeautifulSoup(html_doc, 'lxml')
a_list = soup.select("a") # 获取html页面所有的a标签
for a_s in a_list:
a_s_text = a_s.text.strip("
").strip(" ").strip("
") # 循环获取a标签的text属性并过滤掉
和空格
if filetype == "file":
a_s_text = a_s_text.split("(")[0]
if str(a_s_text) not in dirlist and a_s.parent.parent.name == "tr": # 如果text不等于要保留的类名，则直接删除该节点所属的tr标签
a_s.parent.parent.extract()
del_td = soup.find_all("tr")[0].find_all("td")[1:]
for td in del_td:
td.extract()
# 新增td行Add lines
new_tr = soup.new_tag("td")
new_tr.string = "Add lines"
soup.thead.tr.append(new_tr)
new_tr.attrs = {'class': 'sortable'}
# 新增td行Overlay lines
overlay_tr = soup.new_tag("td")
overlay_tr.string = "Overlay lines"
soup.thead.tr.append(overlay_tr)
overlay_tr.attrs = {'class': 'sortable'}
# 新增td行Coverage
coverage_tr = soup.new_tag("td")
coverage_tr.string = "Coverage"
soup.thead.tr.append(coverage_tr)
coverage_tr.attrs = {'class': 'sortable'}
pack_tr_list = soup.find_all("tbody")[0].find_all("tr") # 获取tbody中tr组成的列表
for tpack in pack_tr_list: # 删除tbody中tr中除类名或文件名的其他列
for pa_td in tpack.find_all("td")[1:]:
pa_td.extract()
tfoot_list = soup.find_all("tfoot")[0].find_all("td")[1:] # 删除tfoot中除Total外的其他列
for tfoot in tfoot_list:
tfoot.extract()
for npack in pack_tr_list:
pack_name = npack.find_all("a")[0].string.strip("
").strip(" ").strip("
")
addlines = 0
covlines = 0
if filetype == "package": # 如果是包名下的index.html文件做如下处理
addlines = ret[pack_name]['new']
covlines = ret[pack_name]['cover']
elif filetype == "root":
for k, v in enumerate(ret[pack_name]):
addlines += ret[pack_name][v]['new']
covlines += ret[pack_name][v]['cover']
elif filetype == "file":
pack_void_name = pack_name.split("(")[0]
filename, diff_dict = kwargs
filename_new_list = filename.split("src/main/java/")[-1].split("/")
filename_new = ".".join(filename_new_list[:-1])
class_name = filename_new_list[-1].split(".")[0]
if filename in diff_dict.keys() and class_name in ret[filename_new].keys():
void_lines_list = diff_dict[filename]['diff_voids'][pack_void_name]
new_line_list = list(
set(ret[filename_new][class_name]['new_lines']).intersection(set(void_lines_list)))
cover_line_list = list(
set(ret[filename_new][class_name]['cover_lines']).intersection(set(void_lines_list)))
addlines = len(new_line_list)
covlines = len(cover_line_list)
if addlines == 0:
coverage = '{:.2%}'.format(0)
else:
coverage = '{:.2%}'.format(covlines / addlines) # 覆盖率
addlines_tr = soup.new_tag("td")
if addlines:
addlines_tr.string = "%s" % addlines
npack.append(addlines_tr)
covlines_tr = soup.new_tag("td")
covlines_tr.string = "%s" % covlines
npack.append(covlines_tr)
coverage_tr = soup.new_tag("td")
coverage_tr.string = "%s" % coverage
npack.append(coverage_tr)
else:
npack.extract()
# 重新生成index.html页面
html_path_new = htmlpath + "_bat"
with open(html_path_new, 'w+') as f:
f.write(HTMLParser.HTMLParser().unescape(soup.prettify()))
os.remove(htmlpath)
os.rename(html_path_new, htmlpath)
2.8 diff_file
循环调用Del_Dr方法执行增量代码覆盖率报告筛选操作；
def diff_file(self):
'''删除多余文件夹并修改index.html文件'''
ret,diff_results = self.process_diff()
diffpaths_list = diff_results.keys()
package_list = ret.keys()
dirs_name = os.listdir(self.coco_path)
del_dirs = list(set(dirs_name).difference(set(package_list)))
for i in del_dirs:
if os.path.isdir(os.path.join(self.coco_path, i)): # 若差集列表中的文件是文件夹则循环删除
re_file = os.path.join(self.coco_path, i)
shutil.rmtree(re_file)
htmlpath = os.path.join(self.coco_path,'index.html')
new_package_list=[]
for pack in package_list:
java_path = os.path.join(self.coco_path,pack)
java_list = []
fina_java_list = []
for java_name in ret[pack].keys():
java_file_name = "src/main/java/" + pack.replace(".","/") + "/" + java_name + ".java"
if java_file_name in diffpaths_list:
javafile_path = os.path.join(java_path,java_name+ '.html')
diff_void_list = diff_results[java_file_name]['diff_voids'].keys()
if len(diff_void_list):
self.Del_Dr(javafile_path,diff_void_list,ret,'file',java_file_name,diff_results)
java_list.append(java_name + '.html')
java_list.append(java_name + '.java.html')
fina_java_list.append(java_name)
new_package_list.append(pack)
fina_java_list = list(set(fina_java_list))
self.Del_Dr(os.path.join(java_path, 'index.html'),fina_java_list, ret[pack],'package')
if len(java_list):
java_list.append("index.html")
java_list.append("index.source.html")
java_dirs = os.listdir(java_path)
del_javas = list(set(java_dirs).difference(set(java_list)))
for d_java in del_javas:
os.remove(os.path.join(java_path,d_java))
new_package_list = list(set(new_package_list))
self.Del_Dr(htmlpath,new_package_list, ret, 'root')
main.py 接收传递过来的参数，调用diff_processor中的类执行筛选操作
import argparse,sys,shutil,os
from diff_processor import DiffProcessor
def main(argv):
#解析命令行参数
parser = argparse.ArgumentParser(description=u"计算增量覆盖率")
parser.add_argument("-dir",type=str,help=u"工程根目录")
parser.add_argument("-giffdir",type=str,help=u"gitdiff文件路径")
parser.add_argument("-jareport",type=str,help=u"jacoco_report路径")
opts = parser.parse_args(argv[1:])
if opts.dir is None or opts.giffdir is None or opts.jareport is None:
parser.print_help()
sys.exit()
#生成增量报告
processor = DiffProcessor(opts.giffdir,opts.dir,os.path.join(opts.jareport,'Check_Order_related'))
processor.diff_file()
#拷贝css和图片资源
shutil.copy('diff.gif', os.path.join(opts.jareport, "jacoco-resources"))
shutil.copy('report.css', os.path.join(opts.jareport, "jacoco-resources"))
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv))
使用方法：
执行命令：
python main.py -d 源码路径 -g diff文件路径 -j jacoco_report路径 如： python main.py -d /root/.jenkins/workspace/Autotest -g /opt/girdiff_1.txt -j /opt/update_report
3.持续集成
3.1 思路
1）jenkins job构建时支持可选择git分支，执行构建，生产gitdiff文件和项目部署；
2）执行手工测试；
3）生成全量覆盖率报告并执行筛选操作；
4）将筛选后的增量覆盖率文件部署到apache下，浏览器访问查看增量报告；
3.2 步骤
1）服务器部署apache并将端口修改为8033；
2）新建部署job1，选择参数化构建git parameter，如下图所示；
3）git源码管理中将分支更改为${new_barnch}；
4）构建执行shell中新增以下代码，其他参照获取之前的文章，部署环境：
echo "开始git diff"
git diff ${old_branch} ${new_branch} >/opt/girdiff_1.txt
echo "git diff已完成，生成增量代码文件，请查看"
5）选择build with parameters，选择old_branch和new_branch，点击执行构建；
6）部署成功后，执行手工测试；
7）复制python_diff文件夹到/opt目录下；（文件夹中包含__init__.py、diff.gif、diff_processor.py、main.py和report.css文件）
8）新建job2，构建选择执行shell，shell内容如下：
cd /opt ant dump ant report rm -rf /opt/update_report #删除增量报告 mkdir /opt/update_report #新建增量文件路径 cp -r /opt/jacoco_report/* /opt/update_report #复制全量报告到update_report目录下 cd python_diff python main.py -d /root/.jenkins/workspace/Autotest -g /opt/girdiff_1.txt -j /opt/update_report #执行增量过滤操作 cd /var/www/html/ rm -rf ./* #删除/var/www/html/目录下的所有文件 cp -r /opt/update_report/* /var/www/html/ chmod -R 755 ./* service httpd restart #重启apache echo "过滤完成，请访问http://ip:8033/index.html查看报告！"
9）访问http://ip:8033/index.html查看报告；