最近一次迭代,参与了公司数据应用平台的开发,其中负责的一块功能早早的就完成了代码的编写工作,即将进入测试阶段,因为有时间思考和总结代码编写中遇到的难题,便想着将代码做一次重构:其中优化的一个功能就是关于数据平台敏感字段的收集
功能描述:数据平台敏感字段的收集:
提供 service 方法,查询是否需要扫描表做触发式收集,指定具体实例与库的表,随机取 N 行(1~max(id) 之间);
a.对每一行的每一个字段的值(取非 null 非空的值)做正则匹配
b. 对每一行的每一个字段的值看是否包含了敏感字段的 key
c. 对每一行的每一个字段名做匹配;如果匹配,再判断该字段为敏感字段还是疑似敏感字段,添加到 secret_column 中
开始的版本:
/** * 敏感字段的收集 * * @param instance * @param schema */ public void collectSecretColumn(String instance, String schema, String table) { //查询该表是否扫描过 CollectedTable collectedTable = collectedTableService.getCollectedTable(instance, schema, table); if (collectedTable != null) { return; } //随机获取n行记录 JdbcResult query = getPartQueryResult(instance, schema, table); if (query == null || (query != null && StringUtils.isNotBlank(query.getQueryErrorMsg()))) { throw new CjjServerException(500, "系统繁忙,请稍后再试"); } //key为column value为值的集合 Map<String, List<String>> groupMap = convertListToMap(query.getResult()); Set<Map.Entry<String, List<String>>> entries = groupMap.entrySet(); List<SecretContainedKey> secretContainedKeys = secretContainedKeyMapper.listSecretContainedKeys(); List<SecretValueRegex> secretValueRegexs = secretValueRegexService.getSecretValueRegexes(); for (Map.Entry<String, List<String>> entry : entries) { //获取column String column = entry.getKey(); List<String> values = entry.getValue(); //判断该字段是否已经存在在敏感字段表中 boolean secretColumnExist = isSecretColumnExist(instance, schema, table, column); if (secretColumnExist) { continue; } //c:对字段名做匹配 boolean isValueContaninsKey = secretContainedKeyService.columnKeyIsContainsKey(instance, schema, table, secretContainedKeys, column); if (isValueContaninsKey) { continue; } //b:字段的值是否包含敏感字段的key boolean isContainsKey = secretContainedKeyService.columnValueIsContainsKey(instance, schema, table, secretContainedKeys, column, values); if (isContainsKey) { continue; } //a:通过正则匹配字段值 secretValueRegexService.regexMatch(instance, schema, table, column, values, secretValueRegexs); } CollectedTable collected = CollectedTable .builder() .instanceName(instance) .schemaName(schema) .tableName(table) .build(); collectedTableMapper.save(collected); }
可以看出逻辑都散落在for循环中
通过责任链模式:后代码:
/** * 敏感字段的收集 * * @param instance * @param schema */ public void collectSecretColumn(String instance, String schema, String table) { //查询该表是否扫描过 CollectedTable collectedTable = collectedTableService.getCollectedTable(instance, schema, table); if (collectedTable != null) { return; } //随机获取n行记录 JdbcResult query = getPartQueryResult(instance, schema, table); if (query == null || (query != null && StringUtils.isNotBlank(query.getQueryErrorMsg()))) { throw new CjjServerException(500, "系统繁忙,请稍后再试"); } //key为column value为值的集合 Map<String, List<String>> groupMap = convertListToMap(query.getResult()); Set<Map.Entry<String, List<String>>> entries = groupMap.entrySet(); secretValueRegexHandler.setSuccessor(secretValueContainedKeyHandler); secretValueContainedKeyHandler.setSuccessor(secretColumnContainedKeyHandler); for (Map.Entry<String, List<String>> entry : entries) { //获取column String column = entry.getKey(); List<String> values = entry.getValue(); //判断该字段是否已经存在在敏感字段表中 boolean secretColumnExist = isSecretColumnExist(instance, schema, table, column); if (secretColumnExist) { continue; } secretValueRegexHandler.handleCollect(instance, schema, table, column, values); } CollectedTable collected = CollectedTable .builder() .instanceName(instance) .schemaName(schema) .tableName(table) .build(); collectedTableMapper.save(collected); }
可以看出这边的代码量减少了,看起来结构更清晰了
为了方便理解:我会列出部分代码供大家参考
package cn.caijiajia.firekylin.service.secret; import java.util.List; /** * 责任链设计模式 * * @author chenlang * date 2018/7/13 */ public abstract class CollectSecretColumnHandler { protected CollectSecretColumnHandler successor; public abstract void handleCollect(String instance, String schema, String table, String column, List<String> values); /** * 获取责任对象 */ public CollectSecretColumnHandler getSuccessor() { return successor; } /** * 设置后继的责任对象 */ public void setSuccessor(CollectSecretColumnHandler successor) { this.successor = successor; } }
package cn.caijiajia.firekylin.service.secret; import cn.caijiajia.firekylin.domain.SecretContainedKey; import cn.caijiajia.firekylin.mapper.SecretContainedKeyMapper; import cn.caijiajia.firekylin.service.SecretColumnService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; /** * @author chenlang * date 2018/7/13 */ @Component public class SecretColumnContainedKeyHandler extends CollectSecretColumnHandler { @Autowired private SecretColumnService secretColumnService; @Autowired private SecretContainedKeyMapper secretContainedKeyMapper; @Override public void handleCollect(String instance, String schema, String table, String column, List<String> values) { List<SecretContainedKey> secretContainedKeys = secretContainedKeyMapper.listSecretContainedKeys(); boolean columnKeyIsContainsKey = columnKeyIsContainsKey(instance, schema, table, secretContainedKeys, column); if (!columnKeyIsContainsKey) { } } public boolean columnKeyIsContainsKey(String instance, String schema, String table, List<SecretContainedKey> secretContainedKeys, String column) { SecretContainedKey secretContainedKeyByColumn = getSecretContainedKeyByColumn(column, secretContainedKeys); if (secretContainedKeyByColumn != null) { secretColumnService.saveSecretColumn(instance, schema, table, column, secretContainedKeyByColumn.getSecretType(), secretContainedKeyByColumn.getColumnType()); return true; } return false; } /** * 字段名是否包含敏感的key * * @param column * @param secretContainedKeys * @return */ public SecretContainedKey getSecretContainedKeyByColumn(String column, List<SecretContainedKey> secretContainedKeys) { Map<String, SecretContainedKey> keysMap = secretContainedKeys.stream().collect(Collectors.toMap(SecretContainedKey::getContainedKey, a -> a)); Set<Map.Entry<String, SecretContainedKey>> entries = keysMap.entrySet(); for (Map.Entry<String, SecretContainedKey> entry : entries) { String key = entry.getKey(); boolean contains = column.toLowerCase().contains(key); if (contains) { return keysMap.get(key); } } return null; } }
package cn.caijiajia.firekylin.service.secret; import cn.caijiajia.firekylin.domain.SecretContainedKey; import cn.caijiajia.firekylin.mapper.SecretContainedKeyMapper; import cn.caijiajia.firekylin.service.SecretColumnService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import java.util.List; /** * @author chenlang * date 2018/7/13 */ @Component public class SecretValueContainedKeyHandler extends CollectSecretColumnHandler { @Autowired private SecretColumnService secretColumnService; @Autowired private SecretContainedKeyMapper secretContainedKeyMapper; @Override public void handleCollect(String instance, String schema, String table, String column, List<String> values) { List<SecretContainedKey> secretContainedKeys = secretContainedKeyMapper.listSecretContainedKeys(); boolean columnValueIsContainsKey = columnValueIsContainsKey(instance, schema, table, secretContainedKeys, column, values); if (!columnValueIsContainsKey) { getSuccessor().handleCollect(instance, schema, table, column, values); } } public boolean columnValueIsContainsKey(String instance, String schema, String table, List<SecretContainedKey> secretContainedKeys, String column, List<String> values) { for (SecretContainedKey secretContainedKey : secretContainedKeys) { boolean isSecretColumnContainsKey = isSecretColumnContainsKey(values, secretContainedKey); if (isSecretColumnContainsKey) { secretColumnService.saveSecretColumn(instance, schema, table, column, secretContainedKey.getSecretType(), secretContainedKey.getColumnType()); return true; } } return false; } /** * 字段值是否包含敏感字段的key * * @param columnValues * @param secretContainedKey * @return */ public boolean isSecretColumnContainsKey(List<String> columnValues, SecretContainedKey secretContainedKey) { for (String columnValue : columnValues) { if (columnValue.toLowerCase().contains(secretContainedKey.getContainedKey())) { return true; } } return false; } }
package cn.caijiajia.firekylin.service.secret; import cn.caijiajia.firekylin.constant.SecretType; import cn.caijiajia.firekylin.domain.SecretValueRegex; import cn.caijiajia.firekylin.service.SecretColumnService; import cn.caijiajia.firekylin.service.SecretValueRegexService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; import java.util.List; import java.util.regex.Pattern; /** * 正则匹配收集敏感字段 * * @author chenlang * date 2018/7/13 */ @Component public class SecretValueRegexHandler extends CollectSecretColumnHandler { @Autowired private SecretColumnService secretColumnService; @Autowired private SecretValueRegexService secretValueRegexService; @Override public void handleCollect(String instance, String schema, String table, String column, List<String> values) { List<SecretValueRegex> secretValueRegexs = secretValueRegexService.getSecretValueRegexes(); boolean regexMatch = regexMatch(instance, schema, table, column, values, secretValueRegexs); if (!regexMatch) { if (getSuccessor() != null) { getSuccessor().handleCollect(instance, schema, table, column, values); } } } public boolean regexMatch(String instance, String schema, String table, String column, List<String> values, List<SecretValueRegex> secretValueRegexs) { for (SecretValueRegex secretValueRegex : secretValueRegexs) { boolean secretByRegex = isSecretByRegex(values, secretValueRegex.getPattern()); if (secretByRegex) { secretColumnService.saveSecretColumn(instance, schema, table, column, SecretType.SECRECT, secretValueRegex.getCode()); return true; } } return false; } /** * 字段值是否匹配正则表达式 * * @param columnValues * @return */ public boolean isSecretByRegex(List<String> columnValues, Pattern compile) { if (CollectionUtils.isEmpty(columnValues)) { return false; } for (String columnValue : columnValues) { boolean isSecret = compile.matcher(columnValue).matches(); if (!isSecret) { return false; } } return true; } }
现在每种情况对应一种handler,同时继承自
CollectSecretColumnHandler