Node.js消耗来自外部API的deflate响应

Question

This should be an easy question but for the sake of my life I cannot make it work, I'm consuming a web service like this:

var XMLHttpRequest = require('XMLHttpRequest').XMLHttpRequest;
var xhr = new XMLHttpRequest();
xhr.open('GET', 'https://rest.gestionix.com/api/v2/products? 
branch_id=7471&filter=0119080PMDSV&results_per_page=5&page=1&fields=id');
xhr.setRequestHeader("Content-Type", "application/json; charset=utf-8");
xhr.onreadystatechange = function(event) {
    if (xhr.readyState === 4) {
        if (xhr.status === 200) {
            console.log(xhr.responseText);
        }
    }
    console.log(xhr.getAllResponseHeaders());
};
xhr.setRequestHeader('Accept','application/json');
xhr.setRequestHeader('Accept-Encoding','decode');
xhr.setRequestHeader('Content-Encoding','decode');
xhr.setRequestHeader('Encoding','decode');
xhr.setRequestHeader('apikey', '---'); <<< of course I'm using an apikey
xhr.send();

The api returns this header:

cache-control: max-age=60
content-length: 22766
content-type: application/json
content-encoding: deflate
server: Microsoft-IIS/10.0
x-aspnet-version: 4.0.30319
x-powered-by: ASP.NET
date: Mon, 02 Jul 2018 16:31:32 GMT
connection: close

However, the content is just a bunch of weird chars:

a ^G4Wk wC p Ȳ G ?FZ} Ϧ Bo W i gu $H ^; ,Wf 촞 } : P e yE %6٬e1D ml 7UO DzK m }t " u dS7 Q >5 y֫ I ;E PH } / X & W{ )X SP v [ ݰ k W׈ P{ W >Z י R ׺4T ]X m< Ns'՟ f 0X:V W C ҁ P #d T gb yI n c- +EP #=| V f 9 Ղ h : r yF ر Se !σr L/E d7 7 \\ +ɠ N 3 a { - ) ~ . \\s ^5 q .t & Ǧ oP - ;( 4 o6

I have tried with different encodings but the result is always the same, I have look for documentation on how to decompress this but I have not find anything that works, if anyone have a link that can point me in the right direction I'll really appreciate it.

Answer 1

Server probably ignores your request to non uncompressed data and sends you compressed data. It uses deflate algorithm as said in content-encoding header. You can deflate it yourself. nodejs has native library zlib which can deflate data.

var zlib = require("zlib")

// in response callback
zlib.deflate(xhr.response, function (error, result) {
    console.log(result.toString());
});

In my case I was unable to read data XMLHttpRequest because this library cannot process binary data and it process all data as strings. This is problem because converting some binary characters (like 0) breaks the data. I have to switched to request library instead. Code I was using to test is

var zlib = require("zlib")
var request = require("request")

request('http://something...', { encoding: null }, function (error, response, body) {
    zlib.deflate(xhr.response, function (error, result) {
        console.log(result.toString());
    });
});

Answer 2

Well, I didn't wanted to let this unanswered just in case someone else needs it. First you need this boiler plate code

/** Boiler Plate Code * Process the response. * @param {Object} headers * A hash of response header name-value pairs. * @param {String} body * The uncompressed response body. */ function processResponse(headers, body) {

  **do something with the response**

    } else {
        console.log("Response is empty.");
    }
}

/**
 * Manage an error response.
 *
 * @param {Error} error
 *   An error instance.
 */
function handleError(error) {
    // Error handling code goes here.
    console.log("Error Found: " + error);

}

/**
 * Obtain the encoding for the content given the headers.
 *
 * @param {Object} headers
 *   A hash of response header name-value pairs.
 * @return {String}
 *   The encoding if specified, or 'utf-8'.
 */
console.log('------------ Getting Charset  ------------');
function obtainCharset(headers) {
    // Find the charset, if specified.
    var charset;
    var contentType = headers['content-type'] || '';
    var matches = contentType.match(/charset=([^;,\r\n]+)/i);
    if (matches && matches[1]) {
        charset = matches[1];
    }
    console.log('------------ Charset is ' + charset + ' (utf-8 if null)  ------------');
    return charset || 'utf-8';
}

This functions will take care of the processing, they go at the top.

And then you need to run your request, in my case I am using a normal var request = require('request-promise');

var req = request({ url: yoururlhere*, headers: **yourheadershere }, function wait(error, response) { if (error) { return handleError(error); } else if (response.statusCode >= 400) { return handleError(new Error(util.format( 'Response with status code %s.', response.statusCode ))); } console.log('----------- Decompressing response -----------'); zlib.inflateRaw(Buffer.concat(buffers), function (gunzipError, bodyBuffer) { if (gunzipError) { return handleError(gunzipError); } var charset = obtainCharset(response.headers); processResponse(response.headers, bodyBuffer.toString(charset)); }); }); req.on('data', function (buf) { buffers[buffers.length] = buf; });