import warnings import logging import os.path import sys import multiprocessing import gensim from gensim.models import Word2Vec from gensim.models.word2vec import LineSentence # 忽略警告 warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim') if __name__ == '__main__': program = os.path.basename(sys.argv[0]) # 读取当前文件的文件名 logger = logging.getLogger(program) logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s',level=logging.INFO) logger.info("running %s" % ' '.join(sys.argv)) # inp为输入语料, outp1为输出模型, outp2为vector格式的模型 inp = 'D:/csv/test/test1.txt' out_model = 'D:/csv/test/test.model' out_vector = 'D:/csv/test/test.vector' # 训练skip-gram模型 model = Word2Vec(LineSentence(inp), size=50, window=5, min_count=5, workers=multiprocessing.cpu_count()) # 保存模型 model.save(out_model) # 保存词向量 model.wv.save_word2vec_format(out_vector, binary=False)
会得到三个文件:.model,.model.syn0.npy,.model.syn1neg.npy,读取就可以:
from gensim.models.deprecated.word2vec import Word2Vec
model = Word2Vec.load('D:/csv/test/text.model') # 3个文件放在一起:.model,.model.syn0.npy,.model.syn1neg.npy
print("read model successful")
另外一种就是获得无后缀的文件,目测这个文件包含了这三个文件的。
import warnings
import gensim
from gensim.models import Word2Vec
from gensim.models.word2vec import LineSentence
# 忽略警告
warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim')
filePath = "D:/csv/test/test1.txt" sentences = gensim.models.word2vec.LineSentence(filePath) model = gensim.models.Word2Vec(sentences, size=100, window=5, min_count=1, workers=4) # 训练模型 model.save('D:/csv/test/test1')
读取时:
import gensim inp = 'D:/csv/test/text' # 读取词向量 model = gensim.models.Word2Vec.load(inp)
print("read model successful")