按行读取文本,去重行

readDir = '去重文本.txt'
writeDir = '新文本.txt'  # 结果文件的路径
lines_seen = set() #空集合,集合的特性,去重
outfile=open(writeDir,"w+",encoding='utf-8-sig')
f = open(readDir,"r",encoding='utf-8-sig')
for line in f:
    if line not in lines_seen:
        outfile.write(line)
        lines_seen.add(line)
outfile.close()
f.close()