Save array of packed record to disk

Question

I am getting crazy. I am trying to save an array of packed record to disk to be read later on.

The following unit contain mainly two procedures.

1. The first one InitSaveAndReLoad() , init a packed array of record, save it to the disk and reload from the disk in a new array of packed record and go through the loaded array and print the 20 first value. Works perfectly.
2. The second one LoadFromFile(), just reload the array from disk. It is even call by InitSaveAndReload() and works perfectly as soon as the file has been created previously by the same instance of the application . I mean if I quit the application and relaunch, the LoadFromFile() procedure which just reload the file in an array of records does not works anymore. I don't understand why.

Any clue?

Thanks for your help. Already spend a full day on this issue and turning crazy!

unit Unit4;

interface

uses
  Winapi.Windows, Winapi.Messages, System.SysUtils, System.DateUtils,
  System.Variants, System.Classes, Vcl.Graphics,
  Vcl.Controls, Vcl.Forms, Vcl.Dialogs, Vcl.Menus, Vcl.StdCtrls;

type
  TRate = packed record
    time        : int64;
    open        : double;
    low         : double;
    high        : double;
    close       : double;
    tick_volume : int64;
    spread      : integer;
    real_volume : int64;
  end;

  PRate = ^TRate;

  TForm4 = class(TForm)
    MemoLogs: TMemo;
    SaveDialog1: TSaveDialog;
    edFile: TEdit;
    Button1: TButton;
    Button2: TButton;
    Button3: TButton;
    Button4: TButton;
    procedure InitSaveAndReload(Sender: TObject);
    procedure Reload(Sender: TObject);
    procedure SelectFile(Sender: TObject);
    procedure LoadFromFile();
    procedure Button4Click(Sender: TObject);
  private
    { Private declarations }
  public
    { Public declarations }
  end;

var
  Form4: TForm4;

implementation

{$R *.dfm}

function TimeElapsedToString(time : int64; show_ms : boolean = false) : string;
var
  TmpVal:real;
  TmpStr:string;
begin
  TmpVal := time;
  TmpStr := '';

  TmpVal := TmpVal / 3600000;
  TmpStr := inttostr(trunc(TmpVal));
  if Length(TmpStr) = 1 then TmpStr := '0' + TmpStr;


  TmpVal := (TmpVal-trunc(TmpVal))* 3600000;
  TmpVal := TmpVal / 60000;
  if TmpVal<10 then
     TmpStr := TmpStr + ':0' + inttostr(trunc(TmpVal))
  else
     TmpStr := TmpStr + ':' + inttostr(trunc(TmpVal));

  TmpVal := (TmpVal-trunc(TmpVal))*60000;
  TmpVal := TmpVal / 1000;
  if TmpVal<10 then
     TmpStr := TmpStr + ':0' + inttostr(trunc(TmpVal))
  else
     TmpStr := TmpStr + ':' + inttostr(trunc(TmpVal));

  if show_ms then
  begin
    TmpVal := (TmpVal-trunc(TmpVal))*1000;
    TmpVal := TmpVal;
    if TmpVal<10 then
      TmpStr := TmpStr + ':00' + inttostr(trunc(TmpVal))
    else if TmpVal<100 then
      TmpStr := TmpStr + ':0' + inttostr(trunc(TmpVal))
    else
      TmpStr := TmpStr + ':' + inttostr(trunc(TmpVal));

  end;

  Result := TmpStr;
end;

procedure TForm4.SelectFile(Sender: TObject);
begin
  if SaveDialog1.Execute then
     edFile.Text := SaveDialog1.FileName;
end;

procedure TForm4.Button4Click(Sender: TObject);
begin
  MemoLogs.Lines.Clear;
end;

procedure TForm4.InitSaveAndReload(Sender: TObject);
var
  _start   : TDatetime;
  ARate    : PRate;
  filename : string;
  Stream   : TFileStream;
  i,L      : integer;
  rates_M1 : array of PRate;
  //rates    : array of PRate;
begin
  filename := edFile.Text;

  MemoLogs.Lines.Add('Initialization of 7 million array of records... Please wait.');
  Refresh;

  // init array
  _start := Now;
  SetLength(rates_M1, 7000000);
  for i:= 0 to 6999999 do
  begin
    New(ARate);
    ARate.time        := DateTimeToUnix(IncMinute(Now, i));
    ARate.open        := 1.25698;
    ARate.low         := 1.2574;
    ARate.high        := 1.2547;
    ARate.close       := 1.65874;
    ARate.tick_volume := 154;
    ARate.spread      := 5;
    ARate.real_volume := 15741;
    rates_M1[i]       := ARate;
  end;
  MemoLogs.Lines.Add(IntToStr(Length(rates_M1)) + ' array of records initialized ' + TimeElapsedToString(MilliSecondsBetween(Now, _start), true));

  // save array
  _start:= Now;
  Stream:= TFileStream.Create(filename , fmCreate);
  try
    L:= Length(rates_M1);
    Stream.WriteBuffer(L, SizeOf(L));
    Stream.WriteBuffer(Pointer(rates_M1)^, L * SizeOf(ARate));
  finally
    Stream.Free;
    MemoLogs.Lines.Add(IntToStr(Length(rates_M1)) + ' records saved to disk in ' + TimeElapsedToString(MilliSecondsBetween(Now, _start), true));
  end;

  LoadFromFile();

end;

procedure TForm4.LoadFromFile;
var
  _start   : TDatetime;
  ARate    : PRate;
  filename : string;
  Stream   : TFileStream;
  i,L      : integer;
  rates    : array of PRate;
begin
  // reload array
  _start := Now;
  filename := edFile.Text;
  Stream:= TFileStream.Create(filename , fmOpenRead);
  try
    Stream.Read(L, SizeOf(L));
    //SetLength(rates_M1, L);
    // even use another empty array of ARate to be sure I am not using the same filled array!
    SetLength(rates, L);
    // I don't want to parse all records...
//    for i := 0 to L-1 do
//      begin
//        Stream.Read(rates_M1[i].AID, SizeOf(ARecord.AID));
//        Stream.Read(rates_M1[i].time, SizeOf(ARecord.time));
//      end;
    Stream.Read(Pointer(rates)^, L * SizeOf(ARate));
  finally
    Stream.Free;
    MemoLogs.Lines.Add(IntToStr(Length(rates)) + ' records loaded from disk in ' + TimeElapsedToString(MilliSecondsBetween(Now, _start), true));
  end;

  // Print 20 first records just reloaded!
  MemoLogs.Lines.Add('Print 20 first records just reloaded in another array of records!' + TimeElapsedToString(MilliSecondsBetween(Now, _start), true));
  for i := 0 to 20 do
     MemoLogs.Lines.Add('i=' + IntToStr(i) + #9
                      + IntToStr(rates[i].time) + #9
                      + FloatToStr(rates[i].open) + #9
                       );

end;

procedure TForm4.Reload(Sender: TObject);
begin
  LoadFromFile();
end;

end.

Result

When I say 'Does not works anymore', I mean once you called InitSaveAndReload() procedure, you can call LoadFromFile() as many time as you want, but if you to call this procedure just after launching the app, trying to use an old file created by the InitSaveAndReload procedure, then it does not work the same!

The Unit provide is as simple as possible. Just create a new projet add 3 buttons a TMemo and one TEdit. If I could join a .rar I would have enclosed the project...

Answer 1

You are saving the address of each record rather than saving contents of the record. You have an array of pointers, and you save that array to file. With your current data structure you would need to save each record individually, because the actual data does not lie contiguously in memory.