backup-analysis-barcode-distribute-recv-py

analysis_barcode.py

---------

# -*- coding:utf-8 -*-

# python3

import sys
import re

# 分析下发和收回的条码
# 1 下发过多少
# 2 回收了多少
# 3 下发过没有回收的有多少
# 4 没下发过但收的有多少

# 下发格式
# barcode=1234567890, machineNo=8, allowToFillCode=1, ctime=2020-07-22 07:27:39

# 回收的格式
# time=2020-07-22 06:18:19, barcode=1234567890, machineNo=2, ...

distribute_file_name_pattern = "RecordBarcodeDistribute-2020-{}.txt"
recv_file_name_pattern = "RecordRecvData-2020-{}.txt"


def read_distributed_info(date):
    '''
    处理下发的数据
    返回文件行数, 和 设备号和向该设备下发过的条码列表的字典
    (rowCount, {"1": ["barcode0", "barcode1", ...], ...})
    '''

    def parse_distribute_row_info(row):
        matchInfo = re.match(r'^barcode=(.{10}), machineNo=(d+), allowToFillCode=(d)', row)
        if matchInfo is None:
            return None
        else:
            return ( matchInfo.group(1), matchInfo.group(2), matchInfo.group(3))

    def append_to(rowInfo, resultDict):
        barcode, machineNo, _ = rowInfo
        if machineNo not in resultDict:
            resultDict[machineNo] = []
        resultDict[machineNo].append(barcode)

    infos = {}
    rowCount = 0

    fileName = distribute_file_name_pattern.format(date)
    with open(fileName, "r") as distributeFile:
        for row in distributeFile:
            rowCount += 1

            parseResult = parse_distribute_row_info(row)
            if parseResult is None:
                print("[WARN] not matched distribute info:", row)
            elif parseResult[2] == "1":
                append_to(parseResult, infos)
            # else ignore

    return (rowCount, infos)



def read_received_info(date):
    """
    处理回收的数据
    返回文件行数, 和 设备号和该设备返回的信息
    (rowCount, {"1", [{"time": "yyyy-MM-dd HH:mm:ss",
                       "barcode": "barcode-value"},
                      ...],
                ...
                } )
    """

    def parse_recv_row_info(row):
        matchInfo = re.match(r'^time=(.{19}), barcode=([^,]+), machineNo=(d+)', row)
        if matchInfo is None:
            return None
        else: 
            return {"machineNo": matchInfo.group(3),
                    "barcode" : matchInfo.group(2),
                    "time": matchInfo.group(1)}

    def append_to(rowInfo, resultDict):
        machineNo = rowInfo["machineNo"]
        if machineNo not in resultDict:
            resultDict[machineNo] = []
        resultDict[machineNo].append({"barcode" : rowInfo["barcode"], "time" : rowInfo["time"]})


    infos = {}
    rowCount = 0

    fileName = recv_file_name_pattern.format(date)
    with open(fileName, "r") as recvFile:
        for row in recvFile:
            rowCount += 1

            parseResult = parse_recv_row_info(row)
            if parseResult is None:
                print("[WARN] not matched recv info:", row)
            else:
                append_to(parseResult, infos)

    return (rowCount, infos)



def calculate_data_count(data):
    result = 0
    for a_list in data:
        result += len(a_list)
    return result

def calculate_count_by_machine(distributeInfo, recvInfo):
    def merge_machine_no_set(machineNosA, machineNosB):
        result = list(set(machineNosA).union(set(machineNosB)))
        result.sort()
        return result

    result = []
    for machineNo in merge_machine_no_set(distributeInfo.keys(), recvInfo.keys()):
        distCount = len(distributedInfo[machineNo]) if machineNo in distributedInfo else 0
        recvCount = len(recvInfo[machineNo]) if machineNo in recvInfo else 0
        result.append({"machineNo": machineNo,
                       "distCount": distCount,
                       "recvCount": recvCount,
                       "diff" : recvCount - distCount})

    return result


def get_repeated_recv_barcodes(recvInfo):
    # 去重
    def exists_in(item, itemList):
        # 判断当前的信息是否与之前的记录重复
        for i in itemList:
            if (i["barcode"] == item["barcode"]) and (i["time"] == item["time"]):
                return True
        return False

    repeatedInfo = []
    deDuplicationInfo = {}

    for k, v in recvInfo.items():
        deDuplicationInfo[k] = []
        for item in v:
            if exists_in(item, deDuplicationInfo[k]):
                repeatedInfo.append((k, item["barcode"], item["time"]))
            else:
                deDuplicationInfo[k].append(item)

    return (deDuplicationInfo, repeatedInfo)


def get_recv_but_not_distributed_barcodes(distributeInfo, recvInfo):
    result = {}
    distributeInfoCopy = {}

    for k, v in distributeInfo.items():
        distributeInfoCopy[k] = v.copy()

    for k, v in recvInfo.items():
        for item in v:
            if item["barcode"] in distributeInfoCopy[k]:
                distributeInfoCopy[k].remove(item["barcode"])
            elif k in result:
                result[k].append(item["barcode"])
            else:
                result[k] = [item["barcode"]]

    return result


def get_distributed_but_not_recv_barcodes(distributeInfo, recvInfo):
    # 下发但是没有回收的条码
    def exists_in(barcode, itemList):
        # 判断下发的条码是否回收
        for item in itemList:
            if barcode == item["barcode"]:
                return True
        return False

    def remove_from(barcode, itemList):
        # 从接收的数据副本中移除对应条码的记录
        targetIdx = None
        for item in itemList:
            if barcode == item["barcode"]:
                target = item
                break

        if target is not None:
            itemList.remove(target)
        else:
            raise Exception("No target exists, in get barcodes distributed but not received.")

    result = {}
    recvInfoCopy = {}

    for k, v in recvInfo.items():
        recvInfoCopy[k] = v.copy()

    for k, v in distributeInfo.items():
        for barcode in v:
            if exists_in(barcode, recvInfoCopy[k]):
                remove_from(barcode, recvInfoCopy[k])
            elif k in result:
                result[k].append(barcode)
            else:
                result[k] = [barcode]

    return result



if __name__ == "__main__":
    # 计算哪个日期的文件, 月日, 04-01
    if len(sys.argv) == 1:
        print("no input, stopped.")
        sys.exit(0)

    date = sys.argv[1]

    # 读取文件
    disFileRowCount, distributedInfo = read_distributed_info(date)
    recvFileRowCount, recvInfo = read_received_info(date)

    # 显示文件总行数
    print("distributed file lines count:", disFileRowCount)
    print("received    file lines count:", recvFileRowCount)

    # 显示下发和回收的总数量
    print("
distributed count:", calculate_data_count(distributedInfo.values()))
    print("received    count:", calculate_data_count(recvInfo.values()))

    # 每个设备的下发和回收数的统计, 及差值
    print("
count by machine:")
    print("machine | distributed-count | recv-count | recv-count - distributed-count")
    for machineCountInfo in calculate_count_by_machine(distributedInfo, recvInfo):
        print("{machineNo:2} | {distCount:4} | {recvCount:4} | {diff:4}".format(**machineCountInfo))

    # 收到的条码有哪些重复, 和去重后的回收结果
    print("
repeated recv barcode:
machine | barcode | time")
    deDuplicationRecvInfo, repeatInfo = get_repeated_recv_barcodes(recvInfo)
    for item in repeatInfo:
        print("{:2} | {:10} | {}".format(*item))
    print("
count:", len(repeatInfo))

    # 去掉重复之后的单设备统计比较
    print("
count by machine after de duplication:")
    print("machine | distributed-count | recv-count | recv-count - distributed-count")
    diffCountInfo = {"distCount" : 0, "recvCount": 0 , "diff" : 0}
    for machineCountInfo in calculate_count_by_machine(distributedInfo, deDuplicationRecvInfo):
        diffCountInfo["distCount"] += machineCountInfo["distCount"]
        diffCountInfo["recvCount"] += machineCountInfo["recvCount"]
        diffCountInfo["diff"] += machineCountInfo["diff"]
        print("{machineNo:5} | {distCount:4} | {recvCount:4} | {diff:4}".format(**machineCountInfo))
    print("count | {distCount:4} | {recvCount:4} | {diff:4}".format(**diffCountInfo))

    # 没下发但是有回收的条码
    print("
received but not distributed barcodes:")
    recvNonDistributedCount = 0
    for k, v in get_recv_but_not_distributed_barcodes(distributedInfo, deDuplicationRecvInfo).items():
        print(k, ": count:", len(v))
        recvNonDistributedCount += len(v)
        for barcode in v:
            print("    [{}]".format(barcode))
    print("count: ", recvNonDistributedCount)

    # 下发后没回收的条码
    print("
distributed but not received barcodes:")
    distributedNonRecvCount = 0
    for k, v in get_distributed_but_not_recv_barcodes(distributedInfo, deDuplicationRecvInfo).items():
        print(k, ": count:", len(v))
        distributedNonRecvCount += len(v)
        for barcode in v:
            print("    [{}]".format(barcode))
    print("count: ", distributedNonRecvCount)

--------- THE END ---------