Mongodb bulk_write & UpdateOne

Using the bulk_write can speed the mongo up, compared with the one by one update or insert.

import pymongo, time, psutil
from concurrent.futures import ProcessPoolExecutor
from pymongo.operations import UpdateOne


tic = time.time()
mongoclient = pymongo.MongoClient(host="mongodb3.xxx.com", port=27017)
MongoDB = mongoclient["xxx"]
collection = MongoDB.resume_test
document = {
    "Duplicate": '',
    "Skill": '',
    "SourceURL": '',
}
document = set(document)

def bulk_write(requests, collection, last_one=False):
    if len(requests) > 10000 or last_one:
        collection.bulk_write(requests)
        return []
    else:
        return requests


requests = []
for index, data in enumerate(collection.find({})):
    if index % 10000 == 0:
        print(index)

    id = data['_id']
    data = set(data)
    add = {el: '' for el in document.difference(data)}
    if add:
        requests.append(UpdateOne({'_id': id}, {'$set': add}))
        requests = bulk_write(requests, collection)
requests = bulk_write(requests, collection, last_one=True)

toc = time.time()
print(f'finished, time cost: {toc - tic}')