Update binary files in Node.js

Question

I'm trying to work with binary files using Node.js.

I'm trying to receive a binary file from the client, open the binary file, convert to hexadecimal, replace data and return the new binary file to the client.

app.use('/read-binary-file',(req,res) => {
    try {
        let content = fs.readFileSync(file_path);
        console.log('content', content)
        res.status(200).send({content})
    }
    catch(err) {
        console.error(err);
    }
})

I wrote code that takes an existing file and try to read it. When I print it, I get this in the buffer:

content <Buffer 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ... >

I'm not sure how to convert it to hexadecimal and then try to change it... When I read the file via online hex editor, I get them in 16 bit each line and this way is very comfortable.

I have some questions:

1. How can convert from binary to hexadecimal?
2. How can I replace data in a hexadecimal file and then return to the client?
3. How can I display them in code as 16 bit?
4. What is the best way to store them in a database? To store the file and then in the database store only the path?

Is there any documentation that can help?

Answer 1

1. Convert from binary to hexadecimal:

    const file = fs.readFileSync("./test"); const str = file.toString("hex");

2. Replace data and return to the client:

    let newStr = str.replace( "00", "FF" ); let buffer = Buffer.from( newStr, "hex" );

3. Display in 16 bits:

    for ( let i = 0; i < newStr.length; i+=16 ){ console.log( newStr.slice( i, i+16 ) + "\\n" ); }

4. Store the file in a writable upload folder, and store the URL path to the uploaded in the database. That's my suggestion, from my experience. You can read more about whether you want to choose to store the images in the database (in BLOB format) in this Quora post: Is it a bad design to store images as blobs in a database?

---

Here's a basic setup that might help you:

/test

ABC

/app.js

const express = require('express');
const app = express();
const fs = require('fs');

app.use("/test", (req, res) => {

    const file = fs.readFileSync("./test");     // Read file as binary
    const str = file.toString("hex");           // Convert to hexadecimal
    let newStr = str.replace(/41|43/g, "42");   // Replace hexadecimal characters
    let buffer = Buffer.from(newStr, "hex");    // Create buffer from hexadecimal

    // Send to the user as download
    res.setHeader('Content-disposition', 'attachment; filename=test-edited.txt');
    res.setHeader('Content-type', 'text/plain');
    res.charset = 'UTF-8';
    res.write(buffer);
    res.end();
});

app.listen(3000, () => console.log('Server Running...'));

The test file contains the characters ABC. They are transformed to BBB and then downloaded.

You can choose to output a different file type, by setting the appropriate filename and MIME type (Content-Type), eg for downloading a PNG image:

    res.setHeader('Content-disposition', 'attachment; filename=output.png');
    res.setHeader('Content-type', 'image/png');

---

Note: for direct binary manipulation, without an intermediary hexadecimal conversion, see Christos Lytras' answer .

Answer 2

You don't need to convert binary data to hexadecimal in order to replace anything in it. If you want to replace data inside a binary file, then it means that you know exactly what you want to replace, which means specific offsets based on some patterns or data structure or byte chunks you'll have to search for.

If you want to create an online hex editor for your clients, then that's a totally different thing and it involves not only backend logic, but also a UI that will let the users upload/edit and download files.

Replacing binary data with binary data on fixed offset using Buffer.copy :

// The first three bytes of data Buffer will be replaced with 0x02 0x03 0x04 bytes
Buffer.alloc(3, new Uint8Array([0x02, 0x03, 0x04])).copy(data, 0, 0, 3);

Using Buffer.indexOf to search for binary data patterns:

// Loading a PNG image
const data = fs.readFileSync('js-logo-16x16.png');

// Search for the PNG signature using Buffer.indexOf
const sigpos = data.indexOf(Buffer.from('PNG'));

if (sigpos >= 0) {
  // If signature pos found (!= -1), replace it with JPG using Buffer.write
  data.write('JPG', sigpos, 3);
}

You can use simple loop and string manipulation logic to print hexadecimal data:

// For Node.js using a Buffer
function displayHexData(data) {
  for (let addr = 0; addr <= data.length; addr += 16) {
    const displayAddr = addr.toString(16).toUpperCase().padStart(8, '0');
    const block = data.slice(addr, Math.min(addr + 16, data.length));

    let hexblock = block.toString('hex');

    if (addr + 16 > data.length) {
      hexblock += '  '.repeat(16 - (data.length - addr));
    }

    const charsblock = Array.from(block)
      .map(b => b >= 32 && b <= 126 ? String.fromCharCode(b) : '.')
      .join('');

    console.log(displayAddr, hexblock.split(/(.{2})/).join(' '), charsblock);
  }
}

will output something like this:

And for web JavaScript:

 // 16 x 16 PNG 8-bit const base64Data = 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQBAMAAADt3eJSAAAAA3NCSVQICAjb4U/gAAAAMFBMVEUAAADQvhqQghFBOgj/7iAjIAT84x+6qxdqYA3u1x2nlxT//CIxLAZKQwnOuhnZyBvQr3QtAAAACXBIWXMAAAsSAAALEgHS3X78AAAAHHRFWHRTb2Z0d2FyZQBBZG9iZSBGaXJld29ya3MgQ1M26LyyjAAAAFNJREFUCJljSIMCBlIZKW4ubmBGhkrtdTDD1Yz5rBuI4XOTyQUs4mPdEA4SSfGx3n7gCZDxKfV+24fVQEYBQyED6zQgI39d2qyVIMUpW9Kyt6UBAGorNUfBuVldAAAAAElFTkSuQmCC'; const data = Uint8Array.from(atob(base64Data), c => c.charCodeAt(0)); displayHexData(data); function displayHexData(data) { let result = ''; for (let addr = 0; addr <= data.length; addr += 16) { const displayAddr = addr.toString(16).toUpperCase().padStart(8, '0'); const block = data.slice(addr, Math.min(addr + 16, data.length)); let hexblock = Array.from (block) .map (b => b.toString(16).toUpperCase().padStart(2, "0")) .join(''); if (addr + 16 > data.length) { hexblock += ' '.repeat(16 - (data.length - addr)); } const charsblock = Array.from(block) .map(b => b >= 32 && b <= 126 ? String.fromCharCode(b) : '.') .join(''); result += `${displayAddr} ${hexblock.split(/(.{2})/).join(' ')} ${charsblock}\\n`; } document.getElementById('hex').appendChild(document.createTextNode(result)); }

 #hex { font-size: small; }

 <pre id="hex"></pre>

As for saving binary data to the database, it depends of course on the database type. Most databases support the Binary Large OBject (BLOB) data type, like MySQL and you use it to store binary data. Saving the file to the filesystem and just using a field to save its path is also a pretty good way to go, but you'll have to handle backups/restores, including those directories where the files will be stored.