zoukankan      html  css  js  c++  java
  • Datawhale-新闻文本分类-task3-机器学习分类

    import pandas as pd
    train = pd.read_csv(r'./train_set.csv', sep='	')
    test_a = pd.read_csv(r'./test_a.csv', sep='	')
    
    from sklearn.feature_extraction.text import CountVectorizer
    from sklearn.model_selection import train_test_split
    from sklearn.linear_model import RidgeClassifier
    def do_nothing_tfidf(train_data):
        import time
        start = time.time()
        from sklearn.feature_extraction.text import TfidfVectorizer
        tfidf = TfidfVectorizer()
        X = tfidf.fit_transform(train['text'])
        y = train['label']
        x_train, x_valid, y_train, y_valid = train_test_split(X, y, test_size=0.3, random_state=seed)
        clf = RidgeClassifier()
        clf.fit(x_train, y_train)
        print(clf.score(x_train,  y_train))
        y_pre = clf.predict(x_valid)
        print(f1_score(y_valid, y_pre, average='macro'))
        print('spent 【%d】s'%(int(time.time() - start)))
    
    def set_ngram_tfidf(train_data):
        import time
        start = time.time()
        from sklearn.feature_extraction.text import TfidfVectorizer
        tfidf = TfidfVectorizer(ngram_range=(1,3))
        X = tfidf.fit_transform(train['text'])
        y = train['label']
        x_train, x_valid, y_train, y_valid = train_test_split(X, y, test_size=0.3, random_state=seed)
        clf = RidgeClassifier()
        clf.fit(x_train, y_train)
        print(clf.score(x_train,  y_train))
        y_pre = clf.predict(x_valid)
        print(f1_score(y_valid, y_pre, average='macro'))
        print('spent 【%d】s'%(int(time.time() - start)))
    
    def set_ngram_max_feature_tfidf(train_data):
        import time
        start = time.time()
        from sklearn.feature_extraction.text import TfidfVectorizer
        tfidf = TfidfVectorizer(ngram_range=(1,3), max_features=5000)
        X = tfidf.fit_transform(train['text'])
        y = train['label']
        x_train, x_valid, y_train, y_valid = train_test_split(X, y, test_size=0.3, random_state=seed)
        clf = RidgeClassifier()
        clf.fit(x_train, y_train)
        print(clf.score(x_train,  y_train))
        y_pre = clf.predict(x_valid)
        print(f1_score(y_valid, y_pre, average='macro'))
        print('spent 【%d】s'%(int(time.time() - start)))
        
    print('=============do nothing tfidf……================')
    do_nothing_tfidf(train)
    print('=============just do ngram tfidf……=============')
    set_ngram_tfidf(train)
    print('=============both do ngram and max_feature……============')
    set_ngram_max_feature_tfidf(train)
    
    

    ![
    ]

  • 相关阅读:
    移动端rem适配
    extern 关键字
    腾讯2014校园招聘软件开发类笔试试题
    堆,栈,堆栈
    转:对TCP/IP网络协议的深入浅出归纳
    转:程序员面试笔试宝典学习记录(一)
    求素数
    [C++]访问控制与继承(public,protect,private) 有时间再整理!!!
    面向对象的static关键字(类中的static关键字)
    腾讯校园招聘会笔试题2011.10.15
  • 原文地址:https://www.cnblogs.com/Alexisbusyblog/p/13378863.html
Copyright © 2011-2022 走看看