K-means 不知k值 自动无监督分类

代码：

# -*- coding:UTF-8 -*-
from numpy import *
import jieba as jb
import time
# 计算权值,并存储为txt
# 计算所有文本包含的总词数
def wordsCount(dataSet):
    wordsCnt = 0
    for document in dataSet:
        wordsCnt += len(document)
    return wordsCnt

# 创建不重复的词条列表
def createVocabList(dataSet):
    vocabSet = set([])
    for document in dataSet:
        vocabSet = vocabSet | set(document)
    return list(vocabSet)

# 将文本转化为词袋模型
def bagOfWords2Vec(vocabList, inputSet):
    returnVec = [0] * len(vocabList)
    for word in inputSet:
        if word in vocabList:
            returnVec[vocabList.index(word)] += 1
        else:
            print("the word: %s is not in my Vocabulary!" % word)
    return returnVec

# 计算包含某个词的文本数
def wordInFileCount(word, cutWordList):
    fileCnt = 0
    for i in cutWordList:
        for j in i:
            if word == j:
                fileCnt = fileCnt + 1
            else:
                continue
    return fileCnt

def calTFIDF(dataSet):
    fileCnt = len(dataSet)  # 文本数
    vocabList = createVocabList(dataSet)  # 词条列表
    tfidfSet = []

    for line in dataSet:
        wordsBag = bagOfWords2Vec(vocabList, line)  # 每行文本对应的词袋向量
        lineWordsCnt = 0
        for i in range(len(wordsBag)):
            lineWordsCnt += wordsBag[i]  # 计算每个文本中包含的总词数
        tfidfList = [0] * len(vocabList)
        for word in line:
            wordinfileCnt = wordInFileCount(word, dataSet)  # 包含该词的文本数
            wordCnt = wordsBag[vocabList.index(word)]  # 该词在文本中出现的次数
            tf = float(wordCnt) / lineWordsCnt
            idf = math.log(float(fileCnt) / (wordinfileCnt + 1))
            tfidf = tf * idf
            tfidfList[vocabList.index(word)] = tfidf
        print(tfidfList)
        print(map(str, tfidfList))
        tfidfSet.append(tfidfList)

    return tfidfSet

# 计算余弦距离
def gen_sim(A, B):
    num = float(dot(mat(A), mat(B).T))
    denum = linalg.norm(A) * linalg.norm(B)
    if denum == 0:
        denum = 1
    cosn = num / denum
    sim = 0.5 + 0.5 * cosn  # 余弦值为[-1,1],归一化为[0,1],值越大相似度越大
    sim = 1 - sim  # 将其转化为值越小距离越近
    return sim


# 计算两个簇的评均距离
def distAvg(dataSet1, dataSet2):
    avgD = 0
    sumD = 0
    m = shape(dataSet1)[0]
    n = shape(dataSet2)[0]
    for i in range(m):
        for j in range(n):
            dist = gen_sim(dataSet1[i], dataSet2[j])
            sumD += dist
    avgD = sumD / (m * n)
    return avgD

# 找到距离最近的两个簇
def findMin(M):
    minDist = inf
    m = shape(M)[0]
    for i in range(m):
        for j in range(m):
            if i != j and M[i, j] < minDist:
                minDist = M[i, j]
                minI = i
                minJ = j
    return minI, minJ, minDist


# 层次聚类算法
def hCluster(dataSet, k, dist, distMeas=distAvg):
    m = shape(dataSet)[0]
    clusterAssment = mat(zeros((m, 1)))
    performMeasure = []
    M = mat(zeros((m, m)))  # 距离矩阵
    # 初始化聚类簇，每个样本作为一个类
    for ii in range(m):
        clusterAssment[ii, 0] = ii

    for i in range(m):
        for j in range(i + 1, m):
            dataSeti = dataSet[nonzero(clusterAssment[:, 0].A == i)[0], :]
            dataSetj = dataSet[nonzero(clusterAssment[:, 0].A == j)[0], :]
            M[i, j] = distMeas(dataSeti, dataSetj)
            M[j, i] = M[i, j]
        if mod(i,10) == 0: print(i)
    q = m  # 设置当前聚类个数
    minDist = 0
    # while (q > k):
    while (minDist < dist):
        i, j, minDist = findMin(M)  # 找到距离最小的两个簇
        # 把第j个簇归并到第i个簇
        clusterAssment[nonzero(clusterAssment[:, 0].A == j)[0], 0] = i
        for l in range(j + 1, q):  # 将j之后的簇重新编号
            clusterAssment[nonzero(clusterAssment[:, 0].A == l)[0], 0] = l - 1
        M = delete(M, j, axis=0)
        M = delete(M, j, axis=1)
        for l in range(q - 1):  # 重新计算第i个簇和其他簇直接的距离
            dataSeti = dataSet[nonzero(clusterAssment[:, 0].A == i)[0], :]
            dataSetl = dataSet[nonzero(clusterAssment[:, 0].A == l)[0], :]
            M[i, l] = distMeas(dataSeti, dataSetl)
            M[l, i] = M[i, l]

        # DBI = DBIvalue(dataSet, clusterAssment, q)
        # DI = DIvalue(dataSet, clusterAssment, q)
        DBI = 0
        DI = 0

        performMeasure.append([q - 1, minDist, DBI, DI])

        q = q - 1

        print(u'当前簇的个数是：', q)
        print(u'距离最小的两个簇是第%d个和第%d个,距离是%f,DBI值是%f,DI值是%f' % (
            i, j, minDist, DBI, DI))

    return clusterAssment, mat(performMeasure)

def saveResult(clusterAssment):
    listResult = clusterAssment.tolist()  # 矩阵转换为list
    for i in range(len(listResult)):
        print(map(str, listResult[i]))


if __name__ =='__main__':
    a=["实施", "效益","节本","10"]
    m=mat(calTFIDF(a))
    clustAssing, performMeasure = hCluster(m, 0, 0.3)
    print(clustAssing)
    saveResult(clustAssing)