在 C++ 中使用 header 读取二进制文件矩阵（这是 fortran 输出）

Question

I have a binary file that has something like a header, and then it's proceeded by the actual matrix. Actually, the binary file is a fortran output, structured like:

      OPEN(UNIT=IUN43,..., FORM='UNFORMATTED',ACCESS='DIRECT',RECL=IRETLEN) 
      WRITE(IUN43,REC=1) "Description text"
      WRITE(IUN43,REC=2) IRECL, ICOLS, IROWS
      WRITE(IUN43,REC=3) I1,I2
      WRITE(IUN43,REC=4) DLAT, DLON, DH
      WRITE(IUN43,REC=5) ITY,ITM,ITD
      WRITE(IUN43,REC=6) I3,I4,I5
      WRITE(IUN43,REC=7) DF
C Loop over IRETREC     
      WRITE(IUN43,17023,REC=IRETREC) I1,I2,D1,D2,D3,...

everything I is integer, D is double. It's very important legacy code, there's nothing I can do about it. REC is different line in the output file. My only experience with binary files is with fortran.

I try to read this in my c++ program. After reading tutorials and answers on Stackoverflow, I finally have something like:

      std::ifstream ifile(TimeSeriesDirectory1+TimeSeriesFileName1, std::ios::binary);
      if (ifile.good())
      {
      // get length of file:
      ifile.seekg (0, ifile.end);
      int filelength = ifile.tellg();
      ifile.seekg (0, ifile.beg);
      char * buffer = new char [filelength];

      std::cout << "Reading " << filelength << " characters...\n";
      std::string line;
      while (std::getline(ifile, line))
      {
       std::istringstream iss(line);
       std::cout << line.substr(0,20) <<std::endl;
       //Only 20 characters to see how and if it works
      };

I tried with ifile.read (line,filelength); and line=const_cast<char*>(buffer); but then the code reads only 1 line, it appears.

My intention is that the code reads the header lines (1-7) that will be copied later on to the output. It could read IRECL, ICOLS, IROWS from the header, since the latter two are the numbers of columns and rows. Currently I'm stuck with just reading the data, since the output is:

    Description text
    ^@^@^@^E^@^@^@y^p2^A^A^@^@^@
    :T   ^b      ^F^\^H         
    ^YN^u4@^xj^#?   ^s?A   |^p^o

etc.

I learnt, that if you know that the file is fixed-size number of doubles, you could use buffer for it. But I don't now columns nor rows before running the program. Also, the idea of having header is very important.

I already have this solved using stringstream (see below for the reference), but it results in 20G-60G matrices, so anything to speed up the process and save the disk space would be appreciated. I use linux (g++ (SUSE Linux) 7.5.0, gfortran GNU Fortran (SUSE Linux) 7.5.0). It came to my mind that maybe I could use casting operators, but it appeared too confusing and I couldn't find out how to do it.

    std::stringstream ss(line);
    while (ss>>number) 
    {
      ++columns;
      LineOfTimeSeriesFile.push_back(number);
    };

Any help is appreciated.

Answer 1

Fun problem. Hope this helps.

Two approaches are below. The first is brute force reading the file. The second approach is helpful if you have fortran code that reads the file.

Approach 1

#include<iostream>
#include<fstream>

using namespace std;

#define F90_REAL_SIZE (sizeof(float))
float read_fortran_real( ifstream &stream )
{
    float tmp = 0.0;
    stream.read( (char*)&tmp, F90_REAL_SIZE );
    return tmp;
}

#define F90_INT_SIZE (sizeof(int))
int read_fortran_int( ifstream &stream )
{
    int tmp = 0;
    stream.read( (char*)&tmp, F90_INT_SIZE );
    return tmp;
}

int main()
{
    int iretlen = 0;
    ifstream stream( "data.raw", ios::out | ios::binary );
    if(!stream) {
        cout << "Cannot open file!" << endl;
        return 1;
    }

    string hdrstr;
    getline( stream, hdrstr, '\0' );
    cout<<"HEADER:"<<hdrstr<<endl;
    iretlen = hdrstr.length() + 1;
    while( stream.peek() == '\0' )
    {
        iretlen++;
        stream.get( );
    }
    cout<<"IRETLEN:"<<iretlen<<endl;

    int irecl = read_fortran_int( stream );
    int icols = read_fortran_int( stream );
    int irows = read_fortran_int( stream );
    stream.seekg( iretlen - 12, ios_base::cur );

    int i1 = read_fortran_int( stream );
    int i2 = read_fortran_int( stream );
    stream.seekg( iretlen - 8, ios_base::cur );

    float dlat = read_fortran_real( stream );
    float dlon = read_fortran_real( stream );
    float  dh = read_fortran_real( stream );
    stream.seekg( iretlen - 12, ios_base::cur );

    int ity = read_fortran_int( stream );
    int itm = read_fortran_int( stream );
    int itd = read_fortran_int( stream );
    stream.seekg( iretlen - 12, ios_base::cur );

    int  i3 = read_fortran_int( stream );
    int  i4 = read_fortran_int( stream );
    int  i5 = read_fortran_int( stream );
    stream.seekg( iretlen - 12, ios_base::cur );

    float  df = read_fortran_real( stream );
    stream.seekg( iretlen - 4, ios_base::cur );

    for ( int i = 0 ; i < 2 ; i++ )
    {
        int i1 = read_fortran_int( stream );
        int i2 = read_fortran_int( stream );
        float d1 = read_fortran_real( stream );
        float d2 = read_fortran_real( stream );
        float d3 = read_fortran_real( stream );
        // 20 bytes exactly
        stream.seekg( iretlen - 20, ios_base::cur );
    }

    stream.close();

    cout<<"IRECL:"<<irecl<<endl;
    cout<<"ICOLS:"<<icols<<endl;
    cout<<"IROWS:"<<irows<<endl;
    cout<<"DLAT:"<<dlat<<endl;
    cout<<"DLON:"<<dlon<<endl;
    cout<<"DH:"<<dh<<endl;

    return 0;
}

Approach 2

Use existing code.

// FILE: try6C.cpp
#include <iostream>

using namespace std;

extern"C" {
void read_header_( char* HDRSTR, int* ICOLS, int* IROWS, int* I1, int* I2, float* DLAT, float* DLON, float* DH, int* ITY, int* ITM, int* ITD, int* I3, int* I4, int* I5, float* DF, float* D1, float* D2, float* D3 );
}

int main()
{
   char HDRSTR[17] = { 0 };
   int ICOLS, IROWS, I1, I2;
   float DLAT, DLON, DH;
   int ITY, ITM, ITD, I3, I4, I5;
   float DF, D1, D2, D3;

   read_header_( HDRSTR, &ICOLS, &IROWS, &I1, &I2, &DLAT, &DLON, &DH, &ITY, &ITM, &ITD, &I3, &I4, &I5, &DF, &D1, &D2, &D3 );

   printf( "%s\n", HDRSTR );
   printf( "%d\n", ICOLS );
   printf( "%d\n", IROWS );

   return 0;
}

! FILE: try6F.f90
subroutine read_header( HDRSTR, ICOLS, IROWS, I1, I2, DLAT, DLON, DH, ITY, ITM, ITD, I3, I4, I5, DF, D1, D2, D3 )
  integer :: IUN43
  integer :: IRECL
  integer, intent(out) :: ICOLS, IROWS
  integer, intent(out) :: I1, I2
  real, intent(out) :: DLAT, DLON, DH
  integer, intent(out) :: ITY, ITM, ITD
  integer, intent(out) :: I3, I4, I5
  real, intent(out) :: DF
  real, intent(out) :: D1, D2, D3
  character(16), intent(out) :: HDRSTR
  IRECL=20
  OPEN( NEWUNIT=IUN43, FILE='data.db', ACCESS='DIRECT', RECL=IRECL, FORM='UNFORMATTED', STATUS='UNKNOWN' ) 
  READ(IUN43,REC=1) HDRSTR
  READ(IUN43,REC=2) IRECL, ICOLS, IROWS
  READ(IUN43,REC=3) I1,I2
  READ(IUN43,REC=4) DLAT, DLON, DH
  READ(IUN43,REC=5) ITY,ITM,ITD
  READ(IUN43,REC=6) I3,I4,I5
  READ(IUN43,REC=7) DF
  close(IUN43)
end subroutine

gfortran -c try6F.f90
g++ -c try6C.cpp 
g++ -o try6 try6C.o try6F.o -lgfortran

Something Extra

A fortran program to create raw binary output.

program hello
  integer :: ILOOP
  integer :: IUN43
  integer :: IRECL, ICOLS, IROWS
  integer :: I1, I2
  real :: DLAT, DLON, DH
  integer :: ITY, ITM, ITD
  integer :: I3, I4, I5
  real :: DF
  real :: D1, D2, D3
  print *, 'Hello, World!'
  IRECL=20
  ICOLS=4
  IROWS=5
  DLAT=1.2
  DLON=3.4
  DH=5.6
  ITY=6
  ITM=7
  ITD=8
  I1=11
  I2=12
  I3=13
  I4=14
  I5=15
  DF=7.8
  D1=10.1
  D2=10.2
  D3=10.3
  OPEN( NEWUNIT=IUN43, FILE='data.db', ACCESS='DIRECT', RECL=IRECL, FORM='UNFORMATTED', STATUS='NEW' ) 
  WRITE(IUN43,REC=1) "Description texta"
  WRITE(IUN43,REC=2) IRECL, ICOLS, IROWS
  WRITE(IUN43,REC=3) I1,I2
  WRITE(IUN43,REC=4) DLAT, DLON, DH
  WRITE(IUN43,REC=5) ITY,ITM,ITD
  WRITE(IUN43,REC=6) I3,I4,I5
  WRITE(IUN43,REC=7) DF
  do ILOOP=8,10
      WRITE(IUN43,REC=ILOOP) I1,I2,D1,D2,D3
  end do 
  close(IUN43)
end program hello