使用 fread 分块读取大文件时数据丢失

Question

我是c++的新手。那是纽约证券交易所的交易数据，我正在用这些数据做量化分析。 一天的单个文件大约是 10G，所以我有数百个这样的文件。 对于第一步，我只想确保我可以正确地按块读取数据。 然后我将操纵那些块。 我被告知使用 python 会很慢，所以我尝试使用 c++。我尝试使用 fread() 以块的形式读取一个文件。 我想计算行数来检查我的代码，但它与我使用记事本++时得到的行数不同。 希望有人能帮我解决这个问题谢谢。

数据：

Q,14340,EUR/NZD,1.65027,1,1.6504,1
T,14340,EUR/NZD,1,1.65034,@,70,X
Q,14340,AUD/NZD,1.03427,1,1.03437,1
T,14340,AUD/NZD,1,1.03432,@,70,X
Q,14340,CAD/CHF,0.75142,1,0.75146,1
T,14340,CAD/CHF,1,0.75144,@,70,X
Q,14340,GBP/NZD,1.90908,1,1.90927,1
T,14340,GBP/NZD,1,1.90918,@,70,X
Q,14340,GBP/CHF,1.312,1,1.31208,1
T,14340,GBP/CHF,1,1.31204,@,70,X
Q,13724,#6S,0.9928,12,0.9929,29

使用 fread 的行数：279 174 248

使用记事本++的行数：279 485 508

错误代码：

#include <iostream>
#include <cstdio>
using namespace std;

int main()
{
    clock_t start = clock();
    char buffer[100000]="\0";
    int cursor = sizeof(buffer);
    FILE* fp;
    int Judge;
    int offset = 0;
    Judge = fopen_s(&fp, "E:\\feedRec\\TFD20190227", "r");
    int count = 0;
    int num = 0;
    while (1)
    {
        //read one chunk
        num=fread(buffer, sizeof(char), (sizeof(buffer) - 1), fp);
        // null terminated the buffer
        buffer[num] = '\0';
        char* ptr=buffer;
        while (*ptr!='\0') {
            if (*ptr == '\n') {
                count++;
            }
            ptr++;
        }
        //Since the lines are not formatted. The buffer will end at
        //the middle of the line. So I need to move back the pointer 
        //back to the beginning of the sliced line to let it be read in 
        //the next loop. 
        //T,14340,AUD/NZD,1,1.03432,@,70,X
        //if buffer end here|
        //pointer need to be move back to the beginning.
        //T,14340,AUD/NZD,1,1.03432,@,70,X
        //|
        //calculte the offset
        offset = 0;
        cursor = sizeof(buffer)-1;
        while (buffer[cursor] != '\n') {
            cursor--;
            offset++;
        }
        //move back the pointer 
        if (offset > 0 && (num == sizeof(buffer) - 1)) {
            fseek(fp, -offset, SEEK_CUR);
        }
        //clear the buffer
        memset(buffer, '\0', sizeof(buffer) - 1);
        // deal with last chunk: if the num less than buffer size, stop
        if (num < sizeof(buffer)-1) 
        {
            fclose(fp); break; 
        }
    }
    //count the last chunk
    char* ptr = buffer;
    while (*ptr != '\0') {
        if (*ptr == '\n') {
            count++;
        }
        ptr++;
    }
    cout << "count: " << count << endl;
    clock_t end = clock();
    cout << "time : " << ((double)end - start) / CLOCKS_PER_SEC << "s\n";
    fclose(fp);
    return 0;
}

Answer 1

有些行有NUL char，用"*ptr:='\0'"判断是否到达缓冲区末尾是错误的：

T,14369,英镑/美元,1,1.33012,@,70,X LF
T,14370,TIME$,1,9453,@,40, NUL LF
Q,10516,#YKH9,901.125,1,911.875,2 LF

循环在“@,40”之后终止。

clock_t start = clock();
char buffer[300]="\0";
int cursor = sizeof(buffer);
FILE* fp;
int Judge;
int offset = 0;
Judge = fopen_s(&fp, "E:\\feedRec\\TFD20190228", "rb");
int count = 0;
size_t num = 0;
while (1)
{
    num=fread(buffer, sizeof(char), (sizeof(buffer) - 1), fp);

    buffer[num] = '\0';


    char* ptr=buffer;
    char* endptr = &buffer[num];
    int ct = 0;
    if (num == sizeof(buffer) - 1) 
    {     
        while (ptr != endptr) {
            ct++;
            if (*ptr == '\n') {
                count++;
            }
            ptr++;
        }
    }
    offset = 0;
    cursor = sizeof(buffer)-1;
    while (buffer[cursor-1] != '\n') {
        cursor--;
        offset++;
    }

    if (offset > 0 && (num == sizeof(buffer) - 1)) {
        fseek(fp, -offset, SEEK_CUR);
    }
    
    if (num < sizeof(buffer)-1) 
    {
        fclose(fp); break; 
    }
    memset(buffer, '\0', sizeof(buffer) - 1);
}

char* ptr = buffer;
char* endptr = &buffer[num];
int ct = 0;
if (num == sizeof(buffer) - 1)
{
    while (ptr != endptr) {
        ct++;
        if (*ptr == '\n') {
            count++;
        }
        ptr++;
    }
}
cout << "count: " << count << endl;
clock_t end = clock();
cout << "time : " << ((double)end - start) / CLOCKS_PER_SEC << "s\n";
fclose(fp);
return 0;