算法：多路归并的外排序

Posted 2021-09-13 滔滔就是我hia

概述：以一个包含很多个整数的大文件为例，来说明多路归并的外排序算法基本思想。假设文件中整数个数为N(N是亿级的)，整数之间用空格分开。首先分多次从该文件中读取M（十万级）个整数，

每次将M个文件在内存中使用快排序之后存入临时文件，然后使用多路归并将临时文件中的数据牌号序存入输出文件。显然，该排序算法需要对每个整数做2次磁盘读和2次磁盘写。

此处只是先实现外排序思想，并未进行性能优化，所以时间复杂度暂不做分析。

ExternSort.h

class ExternSort
{
public:
        void sort()
        {
                //将文件内容分块在内存中排序，并分别写入临时文件
                int file_count = memory_sort();

                //归并临时文件内容到输出文件
                merge_sort(file_count);
        }

        //input_file:输入文件名
        //out_file:输出文件名
        //count: 每次在内存中排序的整数个数
        ExternSort(const char *input_file, const char * out_file, int count)
        {
                m_count = count;
                m_in_file = new char[strlen(input_file) + 1];
                strcpy(m_in_file, input_file);
                m_out_file = new char[strlen(out_file) + 1];
                strcpy(m_out_file, out_file);
        }
        virtual ~ExternSort()
        {
                delete [] m_in_file;
                delete [] m_out_file;
        }

private:
        int m_count; //数组长度
        char *m_in_file;   //输入文件的路径
        char *m_out_file; //输出文件的路径
protected:
        int read_data(FILE* f, int a[], int n)
        {
                int i = 0;
                while(i < n && (fscanf(f, "%d", &a[i]) != EOF)) i++;
                //printf("read:%d integer\\n", i);
                return i;
        }
        void write_data(FILE* f, int a[], int n)
        {
                for(int i = 0; i < n; ++i)
                        fprintf(f, "%d ", a[i]);
        }
        char* temp_filename(int index)
        {
                char *tempfile = new char[100];
                sprintf(tempfile, "temp%d.txt", index);
                return tempfile;
        }
        static int cmp_int(const void *a, const void *b)
        {
                return *(int*)a - *(int*)b;
        }

        int memory_sort()
        {
                FILE* fin = fopen(m_in_file, "rt");
                int n = 0, file_count = 0;
                int *array = new int[m_count];
                //每读入m_count个整数就在内存中做一次排序，并写入临时文件
                while(( n = read_data(fin, array, m_count)) > 0)
                {
                        qsort(array, n, sizeof(int), cmp_int);
                        char *fileName = temp_filename(file_count++);
                        FILE *tempFile = fopen(fileName, "w");
                        free(fileName);
                        write_data(tempFile, array, n);
                        fclose(tempFile);
                }
                delete [] array;
                fclose(fin);

                return file_count;
        }

        void merge_sort(int file_count)
        {
                if(file_count <= 0) return;
                //归并临时文件
                FILE *fout = fopen(m_out_file, "wt");
                //每个文件维持文件指针
                FILE* *farray = new FILE*[file_count];
                int i;
                for(i = 0; i < file_count; ++i)
                {
                        char* fileName = temp_filename(i);
                        farray[i] = fopen(fileName, "rt");
                        free(fileName);
                }

                int *data = new int[file_count];//存储每个文件当前的一个数字
                bool *hasNext = new bool[file_count];//标记文件是否读完
                memset(data, 0, sizeof(int) * file_count);
                memset(hasNext, 1, sizeof(bool) * file_count);
                
                for(i = 0; i < file_count; ++i)
                {
                        if(fscanf(farray[i], "%d", &data[i]) == EOF)//读每个文件的第一个数到data数组
                                hasNext[i] = false;
                }
                int total_num = 0;
                while(true)
                {
                        //求data中可用的最小的数字，并记录对应文件的索引
                        int min_num = data[0];
                        bool bCompare = false;
                        int index = 0;
                        //选择排序
                        for(i = 0; i < file_count; ++i)
                        {    
                            //给min赋一个最初比较值
                            if (!bCompare && hasNext[i])
                            {
                                index = i;
                                min_num = data[i];
                                bCompare = true;
                            }

                            if (bCompare && hasNext[i] && min_num > data[i])
                            {
                                min_num = data[i];
                                index = i;
                            }
                        }
                        if(index == 0 && !hasNext[0]) break; //没有可取的数字，终止归并

                        if (fscanf(farray[index], "%d", &data[index]) == EOF) { //读取文件的下一个元素
                            hasNext[index] = false;
                        }
                        fprintf(fout, "%d ", min_num);
                        total_num++;

                }

                printf("total_num:%d\\n", total_num);
                delete [] hasNext;
                delete [] data;

                for(i = 0; i < file_count; ++i)
                {
                        fclose(farray[i]);
                }
                delete [] farray;
                fclose(fout);
        }
};

ExternSort.cpp

const unsigned int nCount = 100000000; // 需要排序的整数个数

const unsigned int number_to_sort = 10000000; //在内存中一次排序的数量（内存限制整数个数）

const char* unsort_file = "unsort_data.txt"; //原始未排序的文件名

const char* sort_file = "sort_data.txt"; //已排序的文件名

void init_data(unsigned int num) //随机生成数据文件
{
    FILE* f = fopen(unsort_file, "wt");
    for (int i = 0; i < num; ++i)
        fprintf(f, "%d ", rand());
    fclose(f);
}

double getDt(const LARGE_INTEGER& t1, const  LARGE_INTEGER& t2)
{
    double dt;
    LARGE_INTEGER nFreq;
    QueryPerformanceFrequency(&nFreq);
    dt = (t2.QuadPart - t1.QuadPart) / (double)nFreq.QuadPart;
    return dt * 1000;
}

int main(int argc, char** argv)
{
    LARGE_INTEGER t1;
    LARGE_INTEGER t2;
    //time
    QueryPerformanceCounter(&t1);
    srand(time(NULL));
    init_data(nCount);
    ExternSort extSort(unsort_file, sort_file, number_to_sort);
    extSort.sort();
    QueryPerformanceCounter(&t2);
    cout << "deltatime: " << getDt(t1, t2) << " ms" << endl;

    system("pause");
    return 0;
}