Node.JS JSON.parse UTF-8 issue

Question

I've built a small script that allow me to get keywords suggestion from Google search API.

The main problem is if the response contain special characters (like à é ù etc.): my application return me unreadable keywords like that: �,�a,�a va,� majuscule,�a marche,�,�a y est,�a film,�gag,�a il est revenu,�a va de soi,,[object Object]

Here's my Node.js Script:

var express = require('express');
var request = require('request');
var app = express();
app.get('/sug', function (req, res) {
    var KW = req.query.KW ;
    console.time("Délai");
    var url = "http://clients1.google.fr/complete/search?hl=fr&q=" + KW + "&json=t&client=hp";
    request(url, function (err, resp, body) {
        body = JSON.parse(body);
        res.end(body.toString());
        console.timeEnd("Délai");
    });
});
app.listen(1337);
console.log('Serveur : OK');

The call is easy to make, just type http://localhost:1337/sug?KW=ç in your browser.

Do you know how to solve this and get the utf-8 working?

Answer 1

Like vmkcom said, it's because the response is using ISO-8859-1, you have to manually convert to utf-8. iconv package can help you with that:

var request = require('request');
var iconv = require('iconv');

var KW = 'ç' ;
console.time("Délai");
var url = "http://clients1.google.fr/complete/search?hl=fr&q=" + KW + "&json=t&client=hp";

var options = {
    url: url,
    encoding: null // << set encoding to null so request don't try to force utf-8
};

var ic = new iconv.Iconv('iso-8859-1', 'utf-8');

request(options, function (err, resp, body) {
    // body is a Buffer not a string, convert to utf-8 buffer then to utf-8 string
    body = ic.convert(body).toString('utf-8');
    console.log(body);
    console.timeEnd("Délai");
});

Answer 2

Thanks for all the help. So i did this code using your feedbacks:

var express = require('express');
var request = require('request');
var iconv = require('iconv');
var app = express();
app.listen(1337);
console.log('Serveur : OK');
app.get('/sug', function (req, res) {
    var KW = req.query.KW;
    console.time("Délai");
    var url = "http://clients1.google.fr/complete/search?hl=fr&q=" + KW + "&json=t&client=hp";
    var options = {
        url: url,
        encoding: null // << set encoding to null so request don't try to force utf-8
    };
    var ic = new iconv.Iconv('iso-8859-1', 'utf-8');
    request(options, function (err, resp, body) {
        res.set({ 'content-type': 'application/json; charset=utf-8' });
        body = ic.convert(body).toString('utf-8');
        body = JSON.parse(body);
        res.end(body.toString());
        console.timeEnd("Délai");
    });
});

It work great !

http://localhost:1337/sug?KW=%C3%A7a

ça,ça,ça va,ça marche,ça y est,ça film,ça il est revenu,ça va de soi,ça te va,ça s'est bien passé,ça m'énerve,,[object Object]

Answer 3

response set
response.set({ 'content-type': 'application/json; charset=utf-8' });

app.use('/reverse',function (requsting,response) { request.get({ url: url, json: true, headers: {'User-Agent': 'request'} }, (err, res, data) => { if (err) { console.log('Error:', err); } else if (res.statusCode !== 200) {

        response.end(JSON.stringify({status : 'error'}));

    } else {
        response.set({ 'content-type': 'application/json; charset=utf-8' });

        response.end(JSON.stringify({status : 'ok','api' : 'website : https://homeandroid.ir','json':data}));

    }
});

});