Linux C读取文件UNICODE格式的文本（记事本Windows）

Question

Is there a way to read a text file, under Linux with C, saved on Windows as "UNICODE" with notepad? The text in Linux with nano editor looks like:

��T^@e^@s^@t^@
^@

but under vi editor is read properly as:

Test

I must specify the text is normal strings ANSI (no Unicode characters or foreign languages related). Tried like this but no result:

#include <stdio.h>
#include <wchar.h>
#include <locale.h>

int main() {
   char *loc = setlocale(LC_ALL, 0);
   setlocale(LC_ALL, loc);
   FILE * f = fopen("unicode.txt", "r");
   wint_t c;

   while((c = fgetwc(f)) != WEOF) {
      wprintf(L"%lc\n", c);
   }
   return 0;
}

UPDATE:

Forgot to mention the file format is Little-endian UTF-16 Unicode text or UTF-16LE

Answer 1

Include <wchar.h> , set an UTF-8 locale ( setlocale(LC_ALL, "en_US.UTF-8") is fine), open the file or stream in byte-oriented mode ( handle=fopen(filename, "rb") , fwide(handle,-1) , ie in not-wide mode). Then you can use

wint_t getwc_utf16le(FILE *const in)
{
    int lo, hi, code, also;

    if ((lo = getc(in)) == EOF)
        return WEOF;

    if ((hi = getc(in)) == EOF)
        return lo; /* Or abort; input sequence ends prematurely */

    code = lo + 256 * hi;
    if (code < 0xD800 || code > 0xDBFF)
        return code; /* Or abort; input sequence is not UTF16-LE */

    if ((lo = getc(in)) == EOF)
        return code; /* Or abort; input sequence ends prematurely */

    if ((hi = getc(in)) == EOF) {
        ungetc(lo, in);
        return code; /* Or abort; input sequence ends prematurely */
    }

    /* Note: if ((lo + 256*hi) < 0xDC00 || (lo + 256*hi) > 0xDFFF)
     *       the input sequence is not valid UTF16-LE. */
    return 0x10000 + ((code & 0x3FF) << 10) + ((lo + 256 * hi) & 0x3FF);
}

to read code points from such an input file, assuming it contains UTF16-LE data.

The above function is more permissive than strictly necessary, but it does parse all UTF16-LE I could throw at it (including the sometimes problematic U+100000..U+10FFFF code points), so if the input is correct, this function should handle it just fine.

Because the locale is set to UTF-8 in Linux, and Linux implementations support the full Unicode set, the code points match the ones produced by above functions, and you can safely use wide character functions (from <wchar.h> ) to handle the input.

Often the first character in the file is BOM, "byte-order mark" , 0xFEFF . You can ignore it if it is the first character in the file. Elsewhere it is the zero-width non-breaking space. In my experience, those two bytes at the start of a file that is supposed to be text, is quite reliable indicator that the file is UTF16-LE. (So, you could peek at the first two bytes, and if they match those, assume it is UTF16-LE.)

Remember that wide-character end-of-file is WEOF , not EOF .

Hope this helps.

---

Edited 20150505: Here is a helper function one could use instead, to read inputs (using low-level unistd.h interface), converting to UTF-8: read_utf8.h :

#ifndef   READ_UTF8_H
#define   READ_UTF8_H

/* Read input from file descriptor fd,
 * convert it to UTF-8 (using "UTF8//TRANSLIT" iconv conversion),
 * and appending to the specified buffer.
 *    (*dataptr)   points to a dynamically allocated buffer (may reallocate),
 *    (*sizeptr)   points to the size allocated for that buffer,
 *    (*usedptr)   points to the amount of data already in the buffer.
 * You may initialize the values to NULL,0,0, in which case they will
 * be dynamically allocated as needed.
*/
int read_utf8(char **dataptr, size_t *sizeptr, size_t *usedptr, const int fd, const char *const charset);

#endif /* READ_UTF8_H */

read_utf8.c :

#define  _POSIX_C_SOURCE 200809L
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <iconv.h>
#include <string.h>
#include <errno.h>

#define   INPUT_CHUNK  16384
#define   OUTPUT_CHUNK  8192

int read_utf8(char **dataptr, size_t *sizeptr, size_t *usedptr, const int fd, const char *const charset)
{
    char    *data;
    size_t   size;
    size_t   used;

    char    *input_data;
    size_t   input_size, input_head, input_tail;
    int      input_more;

    iconv_t  conversion = (iconv_t)-1;

    if (!dataptr || !sizeptr || !usedptr || fd == -1 || !charset || !*charset)
        return errno = EINVAL;

    if (*dataptr) {
        data = *dataptr;
        size = *sizeptr;
        used = *usedptr;
        if (used > size)
            return errno = EINVAL;
    } else {
        data = NULL;
        size = 0;
        used = 0;
    }

    conversion = iconv_open("UTF8//TRANSLIT", charset);
    if (conversion == (iconv_t)-1)
        return errno = ENOTSUP;

    input_size = INPUT_CHUNK;
    input_data = malloc(input_size);
    if (!input_data) {
        if (conversion != (iconv_t)-1)
            iconv_close(conversion);
        errno = ENOMEM;
        return 0;
    }
    input_head = 0;
    input_tail = 0;
    input_more = 1;

    while (1) {

        if (input_tail > input_head) {
            if (input_head > 0) {
                memmove(input_data, input_data + input_head, input_tail - input_head);
                input_tail -= input_head;
                input_head  = 0;
            }
        } else {
            input_head = 0;
            input_tail = 0;
        }

        if (input_more && input_tail < input_size) {
            ssize_t n;

            do {
                n = read(fd, input_data + input_tail, input_size - input_tail);
            } while (n == (ssize_t)-1 && errno == EINTR);

            if (n > (ssize_t)0)
                input_tail += n;
            else
            if (n == (ssize_t)0)
                input_more = 0;
            else
            if (n != (ssize_t)-1) {
                free(input_data);
                iconv_close(conversion);
                return errno = EIO;
            } else {
                const int errcode = errno;
                free(input_data);
                iconv_close(conversion);
                return errno = errcode;
            }
        }

        if (input_head == 0 && input_tail == 0)
            break;

        if (used + OUTPUT_CHUNK > size) {
            size = (used / (size_t)OUTPUT_CHUNK + (size_t)2) * (size_t)OUTPUT_CHUNK;
            data = realloc(data, size);
            if (!data) {
                free(input_data);
                iconv_close(conversion);
                return errno = ENOMEM;
            }
            *dataptr = data;
            *sizeptr = size;
        }

        {
            char   *source_ptr = input_data + input_head;
            size_t  source_len = input_tail - input_head;

            char   *target_ptr = data + used;
            size_t  target_len = size - used;

            size_t  n;

            n = iconv(conversion, &source_ptr, &source_len, &target_ptr, &target_len);
            if (n == (size_t)-1 && errno == EILSEQ) {
                free(input_data);
                iconv_close(conversion);
                return errno = EILSEQ;
            }

            if (source_ptr == input_data + input_head && target_ptr == data + used) {
                free(input_data);
                iconv_close(conversion);
                return errno = EDEADLK;
            }

            input_head = (size_t)(source_ptr - input_data);
            used = (size_t)(target_ptr - data);

            *usedptr = used;
        }
    }

    free(input_data);
    iconv_close(conversion);

    if (used + 16 >= size) {
        size = (used | 15) + 17;
        data = realloc(data, size);
        if (!data)
            return errno = ENOMEM;
        *dataptr = data;
        *sizeptr = size;
        memset(data + used, 0, size - used);
    } else
    if (used + 32 < size)
        memset(data + used, 0, size - used);
    else
        memset(data + used, 0, 32);

    return errno = 0;
}

and an example program, example.c , on how to use it:

#define  _POSIX_C_SOURCE 200809L
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <string.h>
#include <stdio.h>
#include <errno.h>
#include "read_utf8.h"

int main(int argc, char *argv[])
{
    char   *file_buffer = NULL;
    size_t  file_allocd = 0;
    size_t  file_length = 0;
    int     fd;

    if (argc != 3 || !strcmp(argv[1], "-h") || !strcmp(argv[1], "--help")) {
        fprintf(stderr, "\n");
        fprintf(stderr, "Usage: %s [ -h | --help ]\n", argv[0]);
        fprintf(stderr, "       %s FILENAME CHARSET\n", argv[0]);
        fprintf(stderr, "       %s FILENAME CHARSET//IGNORE\n", argv[0]);
        fprintf(stderr, "\n");
        return EXIT_FAILURE;
    }

    do {
        fd = open(argv[1], O_RDONLY | O_NOCTTY);
    } while (fd == -1 && errno == EINTR);
    if (fd == -1) {
        fprintf(stderr, "%s: %s.\n", argv[1], strerror(errno));
        return EXIT_FAILURE;
    }

    if (read_utf8(&file_buffer, &file_allocd, &file_length, fd, argv[2])) {
        if (errno == ENOTSUP)
            fprintf(stderr, "%s: Unsupported character set.\n", argv[2]);
        else
            fprintf(stderr, "%s: %s.\n", argv[1], strerror(errno));
        return EXIT_FAILURE;
    }

    errno = EIO;
    if (close(fd)) {
        fprintf(stderr, "%s: %s.\n", argv[1], strerror(errno));
        return EXIT_FAILURE;
    }

    fprintf(stderr, "%s: read %zu bytes, allocated %zu.\n", argv[1], file_length, file_allocd);
    if (file_length > 0)
        if (fwrite(file_buffer, file_length, 1, stdout) != 1) {
            fprintf(stderr, "Error writing to standard output.\n");
            return EXIT_FAILURE;
        }

    return EXIT_SUCCESS;
}

This lets you read (either into an empty, dynamically allocated buffer, or append to an existing dynamically allocated buffer) using any character set supported by your system (use iconv --list to see the list), auto-converting the contents to UTF-8.

It uses a temporary input buffer (of INPUT_CHUNK bytes) to read the file part by part, and reallocates the output buffer in multiples of OUTPUT_CHUNK bytes, keeping at least OUTPUT_CHUNK bytes available for each conversion. The constants may need a bit of tuning for different use cases; they're by no means optimal or even suggested values. Larger ones lead to faster code, especially for INPUT_CHUNK , as most filesystems perform better when reading large chunks ( 2097152 is suggested size currently, if I/O performance is important) -- but you should have OUTPUT_CHUNK at similar size, or perhaps twice that, to reduce the number of reallocations needed. (You can trim the resulting buffer afterwards, to used+1 bytes, using realloc() , to avoid memory waste.)