How can I ENABLE_VIRTUAL_TERMINAL_PROCESSING?

Question

Some time ago I noticed that there's a new console mode ENABLE_VIRTUAL_TERMINAL_PROCESSING and I decided to try it out. Here's my sample code:

// File: test1.c
#include <windows.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

const char * const TEST_STRING = "\x1B[31;1mRed\x1B[0m \x1B[32;1mGreen\x1B[0m \x1B[34;1mBlue\x1B[0m";

void ErrorExit(const char* errorMessage) {
    puts(errorMessage);
    exit(1);
}

int main(int argc, char** argv) {
    if (argc != 2) {
        ErrorExit("Usage: program (enable|disable|test|sample)");
    }

    HANDLE hInput = GetStdHandle(STD_INPUT_HANDLE), hOutput = GetStdHandle(STD_OUTPUT_HANDLE);
    DWORD dwMode;

    char *cmd = argv[1];
    if (!strcmp(cmd, "enable")) {
        /*
        GetConsoleMode(hInput, &dwMode);
        dwMode |= ENABLE_VIRTUAL_TERMINAL_INPUT;
        SetConsoleMode(hInput, dwMode);
        */

        GetConsoleMode(hOutput, &dwMode);
        dwMode |= ENABLE_PROCESSED_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING;
        if (!SetConsoleMode(hOutput, dwMode)) {
            ErrorExit("SetConsoleMode failed.");
        }
    }
    else if (!strcmp(cmd, "disable")) {
        /*
        GetConsoleMode(hInput, &dwMode);
        dwMode &= ~ENABLE_VIRTUAL_TERMINAL_INPUT;
        SetConsoleMode(hInput, dwMode);
        */

        GetConsoleMode(hOutput, &dwMode);
        dwMode &= ~ENABLE_VIRTUAL_TERMINAL_PROCESSING;
        if (!SetConsoleMode(hOutput, dwMode)) {
            ErrorExit("SetConsoleMode failed.");
        }
    }
    else if (!strcmp(cmd, "test")) {
        puts(TEST_STRING);
    }
    else if (!strcmp(cmd, "sample")) {
        SetConsoleTextAttribute(hOutput, 0x0C);
        printf("Red");
        SetConsoleTextAttribute(hOutput, 0x07);
        printf(" ");
        SetConsoleTextAttribute(hOutput, 0x0A);
        printf("Green");
        SetConsoleTextAttribute(hOutput, 0x07);
        printf(" ");
        SetConsoleTextAttribute(hOutput, 0x09);
        printf("Blue");
        SetConsoleTextAttribute(hOutput, 0x07);
        printf("\n");
    }
    else {
        ErrorExit("Invalid command!");
    }
    return 0;
}

The code compiled successfully into test1.exe , but it didn't work as expected:

I am fairly sure I've got everything else correct. I'm running latest Windows 10 Enterprise 64-bit, version 10.0.17763.1 .

I have also tried this but it made no difference:

else if (!strcmp(cmd, "test")) {
    DWORD dwNumber = strlen(TEST_STRING), dwWritten;
    WriteConsole(hOutput, TEST_STRING, dwNumber, &dwWritten, NULL);
    puts("");
}

So why isn't my code working (still generating garbage when invoked as test1 test , after running test1 enable )?

Answer 1

The problem is, that the settings you do with SetConsoleMode() affect only the running process (and potential subprocesses). This means, it is not really a setting of the command line window and not "passed back" to the parent shell process. You have to set it directly before you do your output, ie:

else if (!strcmp(cmd, "test")) {
    GetConsoleMode(hOutput, &dwMode);
    dwMode |= ENABLE_PROCESSED_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING;
    if (!SetConsoleMode(hOutput, dwMode)) {
        ErrorExit("SetConsoleMode failed.");
    }
    puts(TEST_STRING);
}

This should work as you expect.