给10^7个有重复的整数排序(败者树)

　　参考July博文：程序员编程艺术：第十章、如何给10^7个数据量的磁盘文件排序，感谢July。

　　给10^7个无重复的整数排序请看另一篇博文：10^7个无重复的整数排序

　　对于给10^7个有重复的整数排序，我们不能用位图法来做，位图法只适用于无重复的数字，那么假设我们没有足够的内存去存储这1千万个整数，我们该如何去排序呢？还是分治法，把大化为小。比如：我们可以把这1千万个整数化为10份，用10个文件存储，分别为data1.txt到data10.txt，并且我们的内存足够存储每一份数据，即每一个dataX.txt，这样，我们就可以依次对这10个文件读取进内存，并利用内部排序，如快速排序，对每一个文件进行排序，然后在对这10个有序的文件进行归并排序，这样就达到我们的要求，即对这10^7个有重复的整数排序了。

　　下面请看代码，我把这10^7个整数分为10份，存储在10个文件中，依次对每一个文件进行快速排序，然后在对这10个文件进行归并排序，在归并的时候，只是采用类似选择排序的方法选择最小值，故比较次数与文件成线性关系。

const int FILE_NUM = 10;
const int MAX_PART = 1000000;
FILE *fpreads[FILE_NUM];

int cmp(const void* a, const void *b)
{
    return *((int*)a) - *((int*)b);
}

//从unsort_data.txt中读取数据
int read_data(FILE *fp, int *array, int N)
{
    int length = 0;
    int num;
    for (int i = 0; i < MAX_PART && (EOF != fscanf(fp, "%d", &num)); i++)
    {
        length++;
        array[i] = num;
    }
    return length;
}

//打开data1.txt - data10.txt这10个文件
FILE* open_file(int count, char *mode)
{
    FILE *fpwrite;
    char filename[20];
    memset(filename, 0, 20);
    sprintf(filename, "data%d.txt", count);
    fpwrite = fopen(filename, mode);
    assert(fpwrite != NULL);
    return fpwrite;
}

//向data1.txt - data10.txt这10个文件写入排好序的数据
void write_data(int *array, int N, int count)
{
    FILE *fpwrite = open_file(count, "w");
    for (int i = 0; i < N; i++)
    {
        fprintf(fpwrite, "%d ", array[i]);
    }
    fclose(fpwrite);
}

//内部排序，调用10次快速排序，产生data1.txt - data10.txt这10个有序文件
void interior_sort(void)
{
    clock_t begin = clock();
    FILE *fpread = fopen("unsort_data.txt", "r");
    assert(fpread != NULL);

    int count = 1;
    int *array = new int[MAX_PART];
    assert(array != NULL);
    while (1)
    {
        memset(array, 0, sizeof(int) * MAX_PART);
        int length = read_data(fpread, array, MAX_PART);
        if (length == 0)
        {
            break;
        }
        qsort(array, length, sizeof(int), cmp);
        write_data(array, length, count);
        count++;
    }
    delete [] array;
    fclose(fpread);
    clock_t end = clock();
    cout<<"10次快速排序所需时间为: "<<(end - begin)/CLK_TCK << "s"<<endl;
}

//对data1.txt - data10.txt这10个有序文件进行归并
void merge_sort()
{
    clock_t begin = clock();
    FILE *fpreads[FILE_NUM];      //10个文件的描述符
    int data[FILE_NUM];           //10个文件的10个当前最小数据
    bool flag[FILE_NUM] = {0};    //标记10个文件，是否已到EOF
    FILE *fpwrite = fopen("sort_data.txt", "w");
    assert(fpwrite != NULL);

    for (int i = 0; i < FILE_NUM; i++)
    {
        fpreads[i] = open_file(i + 1, "r");
    }
    for (int i = 0; i < FILE_NUM; i++)
    {
        fscanf(fpreads[i], "%d", &data[i]);
    }

    while (1)
    {
        int count = 0;
        while (count < FILE_NUM && flag[count])
        {
            count++;
        }
        if (count == FILE_NUM)
        {
            break;
        }
        int min_data = data[count];
        int index = count;
        for (int i = index; i < FILE_NUM; i++)  //在10个文件中找最小的数
        {
            if (!flag[i] && min_data > data[i])
            {
                min_data = data[i];
                index = i;
            }
        }
        fprintf(fpwrite, "%d ", min_data);
        if (EOF == fscanf(fpreads[index],"%d", &data[index]))
        {
            flag[index] = true;
        }
    }
    for (int i = 0; i < FILE_NUM; i++)
    {
        fclose(fpreads[i]);
    }
    fclose(fpwrite);
    clock_t end = clock();
    cout<<"10路归并排序所需时间为: "<<(end - begin)/CLK_TCK << "s"<<endl;
}

int _tmain(int argc, _TCHAR* argv[])
{
    interior_sort();
    merge_sort();
    return 0;
}

　　对于上述归并排序，我们可以用败者树来刷选最小值，这样比较次数就从上述的线性级降到对数级，在归并数多的情况下，效率要比上述的要好，代码如下：

//利用败者树
const int N = 10000000;
const int FILE_NUM = 10;
const int MAX_PART = 1000000;
FILE *fpreads[FILE_NUM];
const int MIN = -1;     //最小值,必须比要排序数字的最小值要小，否则出错
const int MAX = N + 1;  //最大值,必须比要排序数字的最大值要大，否则出错

int cmp(const void* a, const void *b)
{
    return *((int*)a) - *((int*)b);
}

//从unsort_data.txt中读取数据
int read_data(FILE *fp, int *array, int N)
{
    int length = 0;
    int num;
    for (int i = 0; i < MAX_PART && (EOF != fscanf(fp, "%d", &num)); i++)
    {
        length++;
        array[i] = num;
    }
    return length;
}

//打开data0.txt - data9.txt这10个文件
FILE* open_file(int count, char *mode)
{
    FILE *fpwrite;
    char filename[20];
    memset(filename, 0, 20);
    sprintf(filename, "data%d.txt", count);
    fpwrite = fopen(filename, mode);
    assert(fpwrite != NULL);
    return fpwrite;
}

//向data0.txt - data9.txt这10个文件写入排好序的数据
void write_data(int *array, int N, int count)
{
    FILE *fpwrite = open_file(count, "w");
    for (int i = 0; i < N; i++)
    {
        fprintf(fpwrite, "%d ", array[i]);
    }
    fprintf(fpwrite, "%d", MAX);  //在每个文件最后写入一个最大值，表示文件结束
    fclose(fpwrite);
}

//内部排序，调用10次快速排序，产生data0.txt - data9.txt这10个有序文件
void interior_sort(void)
{
    clock_t begin = clock();
    FILE *fpread = fopen("unsort_data.txt", "r");
    assert(fpread != NULL);

    int count = 0;
    int *array = new int[MAX_PART];
    assert(array != NULL);
    while (1)
    {
        memset(array, 0, sizeof(int) * MAX_PART);
        int length = read_data(fpread, array, MAX_PART);
        if (length == 0)
        {
            break;
        }
        qsort(array, length, sizeof(int), cmp);
        write_data(array, length, count);
        count++;
    }
    delete [] array;
    fclose(fpread);
    clock_t end = clock();
    cout<<"10次快速排序所需时间为: "<<(end - begin)/CLK_TCK << "s"<<endl;
}

//调整
void adjust(int ls[], int data[], int s)
{
    int t = (s + FILE_NUM)/2;
    while (t)
    {
        if (data[s] > data[ls[t]])
        {
            int temp = s;
            s = ls[t];
            ls[t] = temp;
        }
        t /= 2;
    }
    ls[0] = s;
}

void create_loser_tree(int ls[], int data[])
{
    data[FILE_NUM] = MIN;
    for (int i = 0; i < FILE_NUM; i++)
    {
        ls[i] = FILE_NUM;
    }
    for (int i = FILE_NUM - 1; i >= 0; i--)
    {
        adjust(ls, data, i);
    }
}

void merge_sort_by_losertree()
{
    clock_t begin = clock();
    FILE *fpreads[FILE_NUM];      //10个文件的描述符
    int data[FILE_NUM + 1];       //10个文件的10个当前最小数据
    int ls[FILE_NUM];             //存放败者索引的节点
    int index;
    FILE *fpwrite = fopen("sort_data_by_losertree.txt", "w");
    assert(fpwrite != NULL);

    for (int i = 0; i < FILE_NUM; i++)
    {
        fpreads[i] = open_file(i, "r");
    }
    for (int i = 0; i < FILE_NUM; i++)
    {
        fscanf(fpreads[i], "%d", &data[i]);
    }

    create_loser_tree(ls, data); //创建败者树
    while (data[ls[0]] != MAX)
    {
        index = ls[0];
        fprintf(fpwrite, "%d ", data[index]);
        fscanf(fpreads[index], "%d", &data[index]);
        adjust(ls, data, index);        
    }
    for (int i = 0; i < FILE_NUM; i++)
    {
        fclose(fpreads[i]);
    }
    fclose(fpwrite);
    clock_t end = clock();
    cout<<"10路归并排序所需时间为: "<<(end - begin)/CLK_TCK << "s"<<endl;
}

int _tmain(int argc, _TCHAR* argv[])
{
    interior_sort();
    merge_sort_by_losertree();
    return 0;
}

　　未排序的数据如下：

　　利用归并排序后的文件如下：

　　2013年1月24日 venow 完