文件循环中的mmap

Question

Im trying to work with mmap reading from file in loop, Im have file contains info about 3 parts, first part is size of 3*sizeof(double), second one also in the size of 3*sizeof(double), and the third with size of sizeof(double). At the first part of the file I have HEADER with size of 32768 bytes. The file organised:

HEADER||Part(1),Part(1)....Part(1)||Part(2),Part(2)....Part(2)||Part(3),Part(3)....Part(3)|

Each part I have 100 times. I want to work each time with 30 parts (10 parts from each part).

I have tried this code:

void readingFile(FILE *file, double *a, double *b, double *c, int start, int end, int chunksz, long total)
{
    int i = 0;
    int size = end - start + 1;
    int fd;
    fd = fileno(file);
    off_t fullsize = lseek(fd,SEEK_CUR,SEEK_END); //getting the file size
    fullsize-=1;//the lseek gives one more byte, its ok!
    unsigned long summ = (unsigned long)(start-1)*chunksz; //chunk is 56
    summ+=(unsigned long)HEADER_SIZE;//offset the header size
    unsigned long paramm=(unsigned long)((unsigned long)summ/(unsigned long)(sysconf(_SC_PAGE_SIZE)));
    unsigned long param = floor(paramm);
    void *buf=NULL;
    buf =mmap(NULL,fullsize , PROT_READ, MAP_PRIVATE , fd, param*sysconf(_SC_PAGE_SIZE));
    if(buf==MAP_FAILED)
    {
        printf("we have an error\n");
    }
    unsigned long gapp = (sysconf(_SC_PAGE_SIZE))*param;
    unsigned long gap =summ-gapp;
    buf+=gap;
    memcpy(a,buf,3*sizeof(double)*size);
    buf+=(unsigned long)((long)total-(start-1))*3*sizeof(double);
    buf+=((start-1)*3*sizeof(double));
    memcpy(b,buf,3*sizeof(double)*size);
    buf+=(unsigned long)((long)total-(start-1))*3*sizeof(double);
    buf+=((start-1)*sizeof(double));
    memcpy(c,buf,sizeof(double)*size);
    munmap(buf, fullsize);
    return;
}

Somewhere in the way I have Overflow and the program crashing! Each time the function being called, A new memory is allotted properly to a,b,c. What is worng here? The process crashed at iteration number 14 in line:

memcpy(c,buf,sizeof(double)*size);

Thanks!

Answer 1

I know that answering a question with source code is not familiar. But i try to expaint what a usefull thing is the mmap. Basically mmap use the kernel capabilities to load (and write back from) the file content into a memory region. so we don't need to frequently call read/seek that can make your application more effective. In other hand it is a confortable solution to access your data directly, just see the code:

#include <unistd.h>
#include <sys/mman.h>
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

struct mapping
{
        void* start_addr;
        size_t length;
        int fd;
};

struct mapping* map_file(const char* file)
{
        struct mapping* ret = malloc(sizeof(struct mapping));
        if(NULL == ret)
        {
                printf("Can't allocate memory for struct mapping.\n");
                return NULL;
        }

        ret->fd = open(file, O_RDONLY);
        if(0 > ret->fd)
        {
                perror("can't open specified file.");
                free(ret);
                return NULL;
        }

        struct stat fs;
        if(0 != fstat(ret->fd, &fs))
        {
                perror("can't specify file size.");
                close(ret->fd);
                free(ret);
                return NULL;
        }

        ret->length = fs.st_size;

        //offset means offset in file
        ret->start_addr = mmap(NULL, ret->length, PROT_READ, MAP_PRIVATE, ret->fd, 0);
        if(MAP_FAILED == ret->start_addr)
        {
                perror("Mapping file failed.");
                close(ret->fd);
                free(ret);
                return NULL;
        }

        return ret;
}

//returns zero on success and free the `struct mapping` data
int unmap_file(struct mapping* mmf)
{
        //note that now we use read only mapping
        //if you want to write this memory pages
        //before detach maybe you have to call:
        //msync(mmf->start_addr, mmf->length, MS_SYNC);
        // avoid data loss (write all dirty page into file).

        if(NULL != mmf->start_addr)
        {
                if(0 != munmap(mmf->start_addr, mmf->length))
                {
                        perror("Can't munmap file.");
                        return 1;
                }
        }

        mmf->start_addr = NULL;
        if(-1 != mmf->fd)
        {
                if(0 != close(mmf->fd))
                {
                        perror("can't close file descriptor.");
                        return 2;
                }
        }

        free(mmf);

        return 0;
}

// for test#define  MAGIC_START_INDEX 0
#define  MAGIC_START_INDEX 32768

int main(int arg_length, char** args)
{
        if(arg_length < 2)
        {
                printf("No input file specified.\n");
                exit(1);
        }

        int i = 0;
        //first argument is the name of program
        while(++i < arg_length)
        {
                struct mapping* mmf = map_file(args[i]);
                if(NULL == mmf)
                {
                        printf("can't use %s for input file\n", args[i]);
                        continue;
                }

                if(mmf->length >  MAGIC_START_INDEX)
                {
                        //upper base
                        int max_index = (mmf->length - MAGIC_START_INDEX) / sizeof(double);

                        //an offset alias for start memory address
                        double* data = ((double*)(mmf->start_addr + MAGIC_START_INDEX));

                        int ni = 0;
                        while(ni+2 < max_index)
                        {
                                printf("num0: %f, num1: %f, num2: %f\n", data[ni], data[ni+1], data[ni+2]);
                                ni += 3;
                        }
                }
                else
                {
                        printf("File: %s has no valuable data.", args[i]);
                }

                unmap_file(mmf);
        }
}

In the main, you see, we can use directly memory addresses instead of repeat read operation. This is a sample code what i copied (mapping related data stored in structure and related functions take care about creating/releace file mapping). That reading can be more lazy, just open the file, read the size (fstat) if i has valuable data, use mmap's offset parameter to skip the header section in the file:

double[] data = (double*)(mmap(NULL, file_length, PROT_READ, MAP_PRIVATE, fd, MAGIC_START_INDEX)); //TODO check null.

and you get "an instant" access to the data.