C: never getting to return statement

Question

I am implementing a program which hides messages in ppm files and then retrieves them. One of the last things I need to do is to "return 2" if there is a problem opening the file. If I enter a file which can't be opened (file doesn't exist or wrong extension, etc) then it will display the error message on stderr, but for some reason it won't execute the very next line, "return 2"!

It simply then returns 0 instead of returning 2.

int load_ppm_image(const char *input_file_name, unsigned char **data, int *n,
               int *width, int *height, int *max_color)
{
    char line[80];
    char c;

    FILE * image_file;
    //image_file = fopen(input_file_name, "rb");

    if (fopen(input_file_name, "rb") == NULL)  // checks to see if file cant be opened
    {
        fprintf(stderr, "The input image file could not be opened\n");
        return 2;  // why isn't this being returned???
    }
    else
    {
        image_file = fopen(input_file_name, "rb");
    }

    fscanf(image_file, "%s", line);

    fscanf(image_file, "%d %d", width, height);

    *n = (3 * (*width) * (*height));

    fscanf(image_file, "%d%c", max_color, &c);
    *data = (unsigned char *)malloc((*n)*sizeof(unsigned char));
    size_t m = fread(*data, sizeof(unsigned char), *n, image_file);

    assert(m == *n);
    fclose(image_file);
    return 0;
}

int hide_message(const char *input_file_name, const char *message, const char *output_file_name)
{
    unsigned char * data;
    int n;
    int width;
    int height;
    int max_color;

    n = 3 * width * height;
    int code = load_ppm_image(input_file_name, &data, &n, &width, &height, &max_color);

    if (code)
    {
        // return the appropriate error message if the image doesn't load correctly
        return code;
    }

    int len_message;
    int count = 0;
    unsigned char letter;

    // get the length of the message to be hidden
    len_message = (int)strlen(message);

    for(int j = 0; j < len_message; j++)
    {

        letter = message[j];
        int mask = 0x80;

        // loop through each byte
        for(int k = 0; k < 8; k++)
        {
            if((letter & mask) == 0)
            {
                //set right most bit to 0
                data[count] = 0xfe & data[count];
            }
            else
            {
                //set right most bit to 1
                data[count] = 0x01 | data[count];
            }
            // shift the mask
            mask = mask>>1 ;
            count++;
        }
    }
    // create the null character at the end of the message (00000000)
    for(int b = 0; b < 8; b++){
        data[count] = 0xfe & data[count];
        count++;
    }

    // write a new image file with the message hidden in it
    int code2 = write_ppm_image(output_file_name, data, n, width, height, max_color);

    if (code2)
    {
        // return the appropriate error message if the image doesn't load correctly
        return code2;
    }

    return 0;
}

---

Edit:

int main(int argc, const char * argv[])
{
    if (argc < 2 || argv[1][0] != '-')
    {
        // the user didn't enter a switch
        fprintf(stderr, "Usage: no switch was entered.\n");
        return 1;
    }

    else if ((strcmp(argv[1], "-e") == 0) && (argc == 5))
    {
        // hides a message in an output file
        hide_message(argv[2], argv[3], argv[4]);
    }

    else if ((strcmp(argv[1], "-d") == 0) && (argc == 3))
    {
        // retrieves a hidden message in a given file
        retrieve_message(argv[2]);
    }

    else
    {
        // all other errors prompt a general usage error
        fprintf(stderr, "Usage: error");
        return 1;
    }
}

Answer 1

Program ended with exit code: 0

The exit code of a C program is the return value of main or the status value in an explicit exit function call. In your case, the load_ppm_image returns a value to its caller function hide_message which in turn returns the same value to its caller function main . However, main does not explicitly return anything for the case that calls hide_message . The C standard specifies that main implicitly returns 0 if it reaches the end of the function without an explicit return. Hence the exit code of 0 in your case.

To get the behaviour you desire modify your main code to return the hide_message return value.

else if ((strcmp(argv[1], "-e") == 0) && (argc == 5))
{
    // hides a message in an output file
    return (hide_message(argv[2], argv[3], argv[4]));
}

Answer 2

From code :

In main function you are calling the hide_message function, but not collecting return value of hide_message()

hide_message(argv[2], argv[3], argv[4]);

So the main function is ending with return 0.