Creating a CNN model for multi-output prediction where one target variable is categorical, and others are numeric

Question

I want to create a simple CNN model for multi-output prediction. The predicted values are four numeric values (all between 0-1) and one categorical value (4 classes). When I try to create a model using Keras, I cannot predict numeric and categorical values using one model.

I use a workaround for a categorical variable, where I one-hot-encode it and predict one-hot-encoded values. What would be the right approach for multi-output prediction, where the predicted values are categorical and numeric; How do I change the code to use the Categorical Cross entropy loss function and argmax to predict the categorical output variable (simultaneously with other numerical output variables)?

from tensorflow.keras.utils import to_categorical
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras import layers
import tensorflow.keras as keras
import pandas as pd


def baselineCNNModel(train_df, test_df, OUTPUT_DIR_TRAIN, OUTPUT_DIR_TEST, debug = False):
    """
        Create a baseline CNN model for multi-output prediction. 
        The input is full images (containing one or more road signs). 
        The target prediction values are class labels and bounding box information. 
    """

    print("\nrunSimpleModel\n")
    train_dataset = train_df[['Class Number', 'Center in X', 'Center in Y', 'Width', 'Height', 'Image Filename']]
    test_dataset = test_df[['Class Number', 'Center in X', 'Center in Y', 'Width', 'Height', 'Image Filename']]

    train_class_number_labels_one_hot = to_categorical(train_dataset['Class Number'], num_classes = 4)
    test_class_number_labels_one_hot = to_categorical(test_dataset['Class Number'], num_classes = 4)
    # Add one-hot encoded columns to the DataFrame
    for i in range(4):
        train_dataset[f'Class Number {i}'] = train_class_number_labels_one_hot[:, i]
        test_dataset[f'Class Number {i}'] = test_class_number_labels_one_hot[:, i]

    print("train_df: ")
    print(train_dataset)
    print("\ntest_df: ")
    print(test_dataset)
    
    tDIR, sDIR = OUTPUT_DIR_TRAIN, OUTPUT_DIR_TEST
    BS, image_size = 64, (128, 128) # batch size; image dimensions required by pretrained model

    # Data preprocessing and augmentation
    datagen = ImageDataGenerator(
        rescale = 1.0 / 255.0,
        validation_split = 0.2
    )
    train_generator = datagen.flow_from_dataframe(
        dataframe = train_dataset,
        directory = tDIR,
        x_col = "Image Filename", # Column containing image filenames
        # y_col = ["Class Number", "Center in X", "Center in Y", "Width", "Height"],
        y_col = ["Class Number 0", "Class Number 1", "Class Number 2", "Class Number 3", "Center in X", "Center in Y", "Width", "Height"],
        target_size = image_size,
        batch_size = BS,
        class_mode = 'other',
        subset = 'training'
    )
    validation_generator = datagen.flow_from_dataframe(
        dataframe = train_dataset,
        directory = tDIR,
        x_col = "Image Filename",
        # y_col = ["Class Number", "Center in X", "Center in Y", "Width", "Height"],
        y_col = ["Class Number 0", "Class Number 1", "Class Number 2", "Class Number 3", "Center in X", "Center in Y", "Width", "Height"],
        target_size = image_size,
        batch_size = BS,
        class_mode='other',
        subset='validation'
    )

    # Define the CNN model
    input_layer = layers.Input(shape = (image_size[0], image_size[1], 3))
    x = layers.Conv2D(128, (4, 4), activation='relu')(input_layer)
    x = layers.MaxPooling2D((4, 4))(x)
    x = layers.Conv2D(64, (3, 3), activation='relu')(x)
    x = layers.MaxPooling2D((2, 2))(x)
    x = layers.Flatten()(x)
    x = layers.Dense(128, activation='relu')(x)
    
    # Create separate heads for each label
    class_number_head = layers.Dense(1, activation="sigmoid", name='class_number')(x)
    class_number_head1 = layers.Dense(1, activation="sigmoid", name='class_number1')(x)
    class_number_head2 = layers.Dense(1, activation="sigmoid", name='class_number2')(x)
    class_number_head3 = layers.Dense(1, activation="sigmoid", name='class_number3')(x)
    center_x_head = layers.Dense(1, activation="linear", name='center_x')(x)
    center_y_head = layers.Dense(1, activation="linear", name='center_y')(x)
    width_head = layers.Dense(1, activation="linear", name='width')(x)
    height_head = layers.Dense(1, activation="linear", name='height')(x)

    # Create the multi-output model
    # model = keras.Model(inputs=input_layer, outputs=[class_number_head, center_x_head, center_y_head, width_head, height_head])
    model = keras.Model(inputs=input_layer, outputs=[class_number_head, class_number_head1, class_number_head2, class_number_head3, center_x_head, center_y_head, width_head, height_head])

    # Compile the model with appropriate loss functions and metrics
    model.compile(optimizer='adam',
                loss={'class_number': 'binary_crossentropy',
                      'class_number1': 'binary_crossentropy',
                      'class_number2': 'binary_crossentropy',
                      'class_number3': 'binary_crossentropy',
                      'center_x': 'mean_squared_error', 
                      'center_y': 'mean_squared_error', 
                      'width': 'mean_squared_error', 
                      'height': 'mean_squared_error'},
                metrics={'class_number': 'accuracy',
                         'class_number1': 'accuracy',
                         'class_number2': 'accuracy',
                         'class_number3': 'accuracy', 
                         'center_x': 'mae', 
                         'center_y': 'mae', 
                         'width': 'mae', 
                         'height': 'mae'})

    # Train the model
    epochs = 10
    history = model.fit(train_generator, epochs=epochs, validation_data=validation_generator)

    # Evaluate the model (optional)
    evaluation = model.evaluate(validation_generator)
    print("\nEvaluation Loss:", evaluation)
    print("Evaluation MAE:", evaluation)

    # Make predictions on the test set
    test_datagen = ImageDataGenerator(rescale = 1.0/255.0)
    test_generator = test_datagen.flow_from_dataframe(
        dataframe = test_dataset,
        directory = sDIR,
        x_col = "Image Filename",
        # y_col = ["Class Number", "Center in X", "Center in Y", "Width", "Height"],
        y_col = ["Class Number 0", "Class Number 1", "Class Number 2", "Class Number 3", "Center in X", "Center in Y", "Width", "Height"],
        target_size = image_size,
        batch_size = BS,
        class_mode='other'
    )

    predictions = model.predict(test_generator)
    print(len(predictions[0]))
    print(len(predictions))

    # class_number_predictions, center_x_predictions, center_y_predictions, width_predictions, height_predictions = predictions
    class_number_predictions, class_number_predictions1, class_number_predictions2, class_number_predictions3, center_x_predictions, center_y_predictions, width_predictions, height_predictions = predictions

    # Create a DataFrame
    prediction_df = pd.DataFrame({
        "Class Number 0 ": class_number_predictions.flatten(),
        "Class Number 1": class_number_predictions1.flatten(),
        "Class Number 2": class_number_predictions2.flatten(),
        "Class Number 3": class_number_predictions3.flatten(),
        "Center in X": center_x_predictions.flatten(),
        "Center in Y": center_y_predictions.flatten(),
        "Width": width_predictions.flatten(),
        "Height": height_predictions.flatten(),
        'Image Filename': test_dataset['Image Filename'],
    })

    print("\npredictions: ")
    print(prediction_df) 
```

Answer 1

If I understood you correctly you have two different training objectives:

• CrossEntropy
• MeanSquaredError

Instead of using binary cross entropy I'd suggest you use CategoricalCrossEntropy instead.

You could probably change the code to something like this:

    # Create separate heads for each label
    class_number_head = layers.Dense(4, activation='linear', name='class_number')(x)
    target_head = layers.Dense(1, activation='linear', name='target')(x)
center_x_head = layers.Dense(1, activation="linear", name='center_x')(x)

    # Create the multi-output model
    model = keras.Model(inputs=input_layer, outputs=[class_number_head, target_head])

    # Compile the model with appropriate loss functions and metrics
    model.compile(optimizer='adam',
                loss={'class_number': keras.losses.SparseCategoricalCrossentropy(from_logits=True),
                      'target': 'mean_squared_error'})

The main differences are:

• I've grouped the outputs into classification and regression
• I've changed activation function for classification output to linear, thus also having to specify from_logits=True when initializing the CategoricalCrossEntropy loss
• I'm using SparseCategoricalCrossentropy so you should drop the one hot encoding and use the labels directly instead. You could use CategoricalCrossEntropy loss directly with one hot encoding instead.

I didn't have a dataset on hand so this is completely untested.