use TIdHTTP in Delphi to simulate curl command with user authentication

Question

I am trying to use TIdHTTP in Delphi to simulate the following curl operation:

curl -X POST -F "message={\"user_email\" : \"useremail@domain.com\" , \"user_password\" : \"UserPassword\"}" "https://esm-db.eu/esmws/generate-signed-message/1/query" > token.txt

The guidelines from the server side are given in: https://esm-db.eu/esmws/generate-signed-message/1/query-options.html

The detailed output from a successful curl connection is the following:

Note: Unnecessary use of -X or --request, POST is already inferred.
*   Trying 193.206.88.90...
* TCP_NODELAY set
* Connected to esm-db.eu (193.206.88.90) port 443 (#0)
* schannel: SSL/TLS connection with esm-db.eu port 443 (step 1/3)
* schannel: checking server certificate revocation
* schannel: sending initial handshake data: sending 174 bytes...
* schannel: sent initial handshake data: sent 174 bytes
* schannel: SSL/TLS connection with esm-db.eu port 443 (step 2/3)
* schannel: failed to receive handshake, need more data
* schannel: SSL/TLS connection with esm-db.eu port 443 (step 2/3)
* schannel: encrypted data got 2954
* schannel: encrypted data buffer: offset 2954 length 4096
* schannel: sending next handshake data: sending 93 bytes...
* schannel: SSL/TLS connection with esm-db.eu port 443 (step 2/3)
* schannel: encrypted data got 258
* schannel: encrypted data buffer: offset 258 length 4096
* schannel: SSL/TLS handshake complete
* schannel: SSL/TLS connection with esm-db.eu port 443 (step 3/3)
* schannel: stored credential handle in session cache
> POST /esmws/generate-signed-message/1/query HTTP/1.1
> Host: esm-db.eu
> User-Agent: curl/7.55.1
> Accept: */*
> Content-Length: 217
> Expect: 100-continue
> Content-Type: multipart/form-data; boundary=------------------------7682f54661679429
>
* schannel: client wants to read 102400 bytes
* schannel: encdata_buffer resized 103424
* schannel: encrypted data buffer: offset 0 length 103424
* schannel: encrypted data got 54
* schannel: encrypted data buffer: offset 54 length 103424
* schannel: decrypted data length: 25
* schannel: decrypted data added: 25
* schannel: decrypted data cached: offset 25 length 102400
* schannel: encrypted data buffer: offset 0 length 103424
* schannel: decrypted data buffer: offset 25 length 102400
* schannel: schannel_recv cleanup
* schannel: decrypted data returned 25
* schannel: decrypted data buffer: offset 0 length 102400
< HTTP/1.1 100 Continue
* schannel: client wants to read 102400 bytes
* schannel: encrypted data buffer: offset 0 length 103424
* schannel: encrypted data got 924
* schannel: encrypted data buffer: offset 924 length 103424
* schannel: decrypted data length: 895
* schannel: decrypted data added: 895
* schannel: decrypted data cached: offset 895 length 102400
* schannel: encrypted data buffer: offset 0 length 103424
* schannel: decrypted data buffer: offset 895 length 102400
* schannel: schannel_recv cleanup
* schannel: decrypted data returned 895
* schannel: decrypted data buffer: offset 0 length 102400
< HTTP/1.1 200 OK
< Server: nginx/1.10.3
< Date: Sun, 08 Nov 2020 23:20:51 GMT
< Content-Type: text/plain; charset=UTF-8
< Content-Length: 649
< Connection: keep-alive
< Access-Control-Allow-Origin: *
< Access-Control-Allow-Methods: POST, GET, OPTIONS
<
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA256

However, with TIdHTTP I keep getting an HTTP/1.1 400 Bad Request error. What am I doing wrong?

procedure TMainForm.HTTPGetTokenFile;
var
  IdHTTP: TIdHTTP;
  Params: TIdMultipartFormDataStream;
  LHandler: TIdSSLIOHandlerSocketOpenSSL;
begin
  try
    Params := TIdMultipartFormDataStream.Create;
    Params.AddFormField('message', '{\"user_email\" : \"useremail@domain.com\" , \"user_password\" : \"UserPassword\"}');
    try
      IdHTTP := TIdHTTP.Create(nil);
      try
        LHandler := TIdSSLIOHandlerSocketOpenSSL.Create(IdHTTP);
        LHandler.SSLOptions.Method := sslvTLSv1;
        IdHTTP.IOHandler := LHandler;
        IdHTTP.Request.Accept := 'application/json, text/plain;q=0.9, text/html;q=0.8';
        IdHTTP.Request.ContentType := 'application/json';
        Memo1.Text := IdHTTP.Post('https://esm-db.eu/esmws/generate-signed-message/1/query', Params);
      finally
        IdHTTP.Free;
      end;
    finally
      Params.Free;
    end;
  except
    on E: Exception do
      ShowMessage('Error: ' + E.ToString);
  end;
end;

UPDATE : I removed the \\ characters in the JSON, but I am still getting the HTTP/1.1 400 Bad Request error.

procedure TMainForm.HTTPGetTokenFile;
var
  IdHTTP: TIdHTTP;
  Params: TIdMultipartFormDataStream;
  LHandler: TIdSSLIOHandlerSocketOpenSSL;
begin
  try
    Params := TIdMultipartFormDataStream.Create;
    Params.AddFormField('message', '{"user_email" : "useremail@domain.com" , "user_password" : "UserPassword"}');
    try
      IdHTTP := TIdHTTP.Create(nil);
      try
        LHandler := TIdSSLIOHandlerSocketOpenSSL.Create(IdHTTP);
        LHandler.SSLOptions.Method := sslvTLSv1;
        IdHTTP.IOHandler := LHandler;
        IdHTTP.Request.Accept := 'application/json, text/plain;q=0.9, text/html;q=0.8';
        IdHTTP.Request.ContentType := 'application/json';
        Memo1.Text := IdHTTP.Post('https://esm-db.eu/esmws/generate-signed-message/1/query', Params);
      finally
        IdHTTP.Free;
      end;
    finally
      Params.Free;
    end;
  except
    on E: Exception do
      ShowMessage('Error: ' + E.ToString);
  end;
end;

The dump from TIdHTTP is the following:

Stat Connected.
Sent 9/11/2020 4:36:55 ??: POST /esmws/generate-signed-message/1/query HTTP/1.0<EOL>Content-Type: multipart/form-data; boundary=--------110920163653921<EOL>Content-Length: 257<EOL>Host: esm-db.eu<EOL>Accept: application/json, text/plain;q=0.9, text/html;q=0.8<EOL>Accept-Encoding: identity<EOL>User-Agent: Mozilla/3.0 (compatible; Indy Library)<EOL><EOL>
Sent 9/11/2020 4:36:55 ??: ----------110920163653921<EOL>Content-Disposition: form-data; name="message"<EOL>Content-Type: text/plain<EOL>Content-Transfer-Encoding: quoted-printable<EOL><EOL>{"user_email" : "s.antoniou@seismosoft.com" , "user_password" : "passw=<EOL>ord"}<EOL>----------110920163653921--<EOL>
Recv 9/11/2020 4:36:55 ??: HTTP/1.1 400 Bad Request<EOL>Server: nginx/1.10.3<EOL>Date: Mon, 09 Nov 2020 14:36:56 GMT<EOL>Content-Type: text/html; charset=UTF-8<EOL>Content-Length: 121<EOL>Connection: close<EOL>Access-Control-Allow-Origin: *<EOL>Access-Control-Allow-Methods: POST, GET, OPTIONS<EOL><EOL>{"http_code": 400, "http_label": "Bad Request", "exit_message": "ERROR: improper specification / unrecognized parameter"}
Stat Disconnected.
Stat Disconnected.
Stat Disconnected.

It seems that the arguement of the AddFormField() function has a limit of 70 characters and the final 3 characters seem to be cut off. Can I increase this limit or should I decrease the size of the value passed (eg by removing the unnecessary spaces)?

Answer 1

As Dave mentioned in a comment, you need to remove the \\ characters in your JSON data. Delphi does not escape characters the same way a command line processor does. " is not a reserved character in Delphi, so there is no need to escape a " character in a Delphi string literal.

Also, on a side note, the assignment of IdHTTP.Request.ContentType is redundant in this situation and should be removed. Post() 'ing a TIdMultiPartFormDataStream will overwrite the ContentType with its own value, ignoring whatever you assign.

Try this instead:

procedure TMainForm.HTTPGetTokenFile;
var
  IdHTTP: TIdHTTP;
  Params: TIdMultipartFormDataStream;
  LHandler: TIdSSLIOHandlerSocketOpenSSL;
begin
  try
    Params := TIdMultipartFormDataStream.Create;
    try
      Params.AddFormField('message', '{"user_email" : "useremail@domain.com" , "user_password" : "UserPassword"}');

      IdHTTP := TIdHTTP.Create(nil);
      try
        LHandler := TIdSSLIOHandlerSocketOpenSSL.Create(IdHTTP);
        LHandler.SSLOptions.Method := sslvTLSv1;
        IdHTTP.IOHandler := LHandler;
        IdHTTP.Request.Accept := 'application/json, text/plain;q=0.9, text/html;q=0.8';
        Memo1.Text := IdHTTP.Post('https://esm-db.eu/esmws/generate-signed-message/1/query', Params);
      finally
        IdHTTP.Free;
      end;
    finally
      Params.Free;
    end;
  except
    on E: Exception do
      ShowMessage('Error: ' + E.ToString);
  end;
end;

---

UPDATE: turns out that TIdMultiPartFormDataStream is sending your JSON in Quoted-Printable format, where a "soft" line break is being inserted into the middle of your JSON every 70 characters. Apparently, your server does not support that encoding properly. By default, curl does not use Quoted-Printable for a webform submission unless you explicitly tell it to do so.

You can disable the Quoted-Printable encoding in TIdMultipartFormDataStream by either:

1. setting the TIdFormDataField.ContentTransfer property to either '7bit' , '8bit' , 'binary' , or '' (which is effectively the same as '7bit' but without notifying the server explicitly via a Content-Transfer-Encoding header), eg:

procedure TMainForm.HTTPGetTokenFile;
var
  IdHTTP: TIdHTTP;
  Params: TIdMultipartFormDataStream;
  LHandler: TIdSSLIOHandlerSocketOpenSSL;
begin
  try
    Params := TIdMultipartFormDataStream.Create;
    try
      with Params.AddFormField('message', '{"user_email" : "useremail@domain.com" , "user_password" : "UserPassword"}') do
      begin
        Charset := 'utf-8';
        ContentTransfer := '8bit';
      end;

      IdHTTP := TIdHTTP.Create(nil);
      try
        LHandler := TIdSSLIOHandlerSocketOpenSSL.Create(IdHTTP);
        LHandler.SSLOptions.Method := sslvTLSv1;
        IdHTTP.IOHandler := LHandler;
        IdHTTP.Request.Accept := 'application/json, text/plain;q=0.9, text/html;q=0.8';
        Memo1.Text := IdHTTP.Post('https://esm-db.eu/esmws/generate-signed-message/1/query', Params);
      finally
        IdHTTP.Free;
      end;
    finally
      Params.Free;
    end;
  except
    on E: Exception do
      ShowMessage('Error: ' + E.ToString);
  end;
end;

2. using the AddFormField() overload that takes a TStream instead of a String . You can put your JSON into a TStringStream , eg:

procedure TMainForm.HTTPGetTokenFile;
var
  IdHTTP: TIdHTTP;
  Params: TIdMultipartFormDataStream;
  LHandler: TIdSSLIOHandlerSocketOpenSSL;
  LJSON: TStringStream;
begin
  try
    LJSON := TStringStream.Create('{"user_email" : "useremail@domain.com" , "user_password" : "UserPassword"}', TEncoding.UTF8);
    try
      Params := TIdMultipartFormDataStream.Create;
      try
        with Params.AddFormField('message', 'text/plain', 'utf-8', LJSON) do
          ContentTransfer := '8bit';
    
        IdHTTP := TIdHTTP.Create(nil);
        try
          LHandler := TIdSSLIOHandlerSocketOpenSSL.Create(IdHTTP);
          LHandler.SSLOptions.Method := sslvTLSv1;
          IdHTTP.IOHandler := LHandler;
          IdHTTP.Request.Accept := 'application/json, text/plain;q=0.9, text/html;q=0.8';
          Memo1.Text := IdHTTP.Post('https://esm-db.eu/esmws/generate-signed-message/1/query', Params);
        finally
          IdHTTP.Free;
        end;
      finally
        Params.Free;
      end;
    finally
      LJSON.Free;
    end;
  except
    on E: Exception do
      ShowMessage('Error: ' + E.ToString);
  end;
end;