zoukankan      html  css  js  c++  java
  • 有监督的卡方分箱算法

    实现代码

    import numpy as np
    import pandas as pd
    from collections import Counter
    def chimerge(data, attr, label, max_intervals):
        distinct_vals = sorted(set(data[attr])) # Sort the distinct values
        labels = sorted(set(data[label])) # Get all possible labels
        empty_count = {l: 0 for l in labels} # A helper function for padding the Counter()
        intervals = [[distinct_vals[i], distinct_vals[i]] for i in range(len(distinct_vals))] # Initialize the intervals for each attribute
        while len(intervals) > max_intervals: # While loop
            chi = []
            for i in range(len(intervals)-1):
                # Calculate the Chi2 value
                obs0 = data[data[attr].between(intervals[i][0], intervals[i][1])]
                obs1 = data[data[attr].between(intervals[i+1][0], intervals[i+1][1])]
                total = len(obs0) + len(obs1)
                count_0 = np.array([v for i, v in {**empty_count, **Counter(obs0[label])}.items()])
                count_1 = np.array([v for i, v in {**empty_count, **Counter(obs1[label])}.items()])
                count_total = count_0 + count_1
                expected_0 = count_total*sum(count_0)/total
                expected_1 = count_total*sum(count_1)/total
                chi_ = (count_0 - expected_0)**2/expected_0 + (count_1 - expected_1)**2/expected_1
                chi_ = np.nan_to_num(chi_) # Deal with the zero counts
                chi.append(sum(chi_)) # Finally do the summation for Chi2
            min_chi = min(chi) # Find the minimal Chi2 for current iteration
            for i, v in enumerate(chi):
                if v == min_chi:
                    min_chi_index = i # Find the index of the interval to be merged
                    break
            new_intervals = [] # Prepare for the merged new data array
            skip = False
            done = False
            for i in range(len(intervals)):
                if skip:
                    skip = False
                    continue
                if i == min_chi_index and not done: # Merge the intervals
                    t = intervals[i] + intervals[i+1]
                    new_intervals.append([min(t), max(t)])
                    skip = True
                    done = True
                else:
                    new_intervals.append(intervals[i])
            intervals = new_intervals
        for i in intervals:
            print('[', i[0], ',', i[1], ']', sep='')
    

    使用例子

    iris = pd.read_csv('http://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data', header=None)
    iris.columns = ['sepal_l', 'sepal_w', 'petal_l', 'petal_w', 'type']
    for attr in ['sepal_l', 'sepal_w', 'petal_l', 'petal_w']:
        print('Interval for', attr)
        chimerge(data=iris, attr=attr, label='type', max_intervals=3)
    
    

    结果:

  • 相关阅读:
    Python模糊查询本地文件夹去除文件后缀(7行代码)
    Python正则表达式
    python的logging模块
    Python中hashlib模块
    Python的os模块
    项目初始化mysql建库和授权
    Add correct host key in /root/.ssh/known_hosts to get rid of this message
    高中典型的等比数学题
    autoenv的使用方法
    celery任务进程关闭
  • 原文地址:https://www.cnblogs.com/hichens/p/13585854.html
Copyright © 2011-2022 走看看