How to build a Facial Emotion Detection Model using Deep learning

### Import necessary libraries for numerical operations, deep learning, image processing, and plotting
import numpy as np
import tensorflow as tf
import cv2
import os
import matplotlib.pyplot as plt

### Define paths to the training and testing datasets
trainPath = "C:/Data-Sets/Emotion-Faces/train"
testPath = "C:/Data-Sets/Emotion-Faces/test"

### Get the list of emotion categories from the training folder
folderList = sorted(os.listdir(trainPath))
print(folderList)

### Initialize empty arrays to store training and testing images and labels
X_train, y_train = [], []
X_test, y_test = [], []

### Loop through training data and load each image in grayscale format
for i, category in enumerate(folderList):
    files = os.listdir(os.path.join(trainPath, category))
    for file in files:
        print(f"{category}/{file}")
        img = cv2.imread(os.path.join(trainPath, category, file), cv2.IMREAD_GRAYSCALE)
        X_train.append(img)
        y_train.append(i)  # Assign numeric label

print(f"Number of training images: {len(X_train)}")

### Load testing data in the same way as training data
folderList = sorted(os.listdir(testPath))
for i, category in enumerate(folderList):
    files = os.listdir(os.path.join(testPath, category))
    for file in files:
        print(f"{category}/{file}")
        img = cv2.imread(os.path.join(testPath, category, file), cv2.IMREAD_GRAYSCALE)
        X_test.append(img)
        y_test.append(i)

print(f"Number of test images: {len(X_test)}")

### Convert lists to numpy arrays for better performance
X_train = np.array(X_train, dtype='float32')
y_train = np.array(y_train, dtype='float32')
X_test = np.array(X_test, dtype='float32')
y_test = np.array(y_test, dtype='float32')

### Normalize image pixel values to the range 0-1
X_train /= 255.0
X_test /= 255.0

### Reshape images to include channel dimension (48x48x1)
X_train = X_train.reshape(-1, 48, 48, 1)
X_test = X_test.reshape(-1, 48, 48, 1)

### Convert emotion labels into categorical format (7 classes)
num_classes = 7
y_train = tf.keras.utils.to_categorical(y_train, num_classes=num_classes)
y_test = tf.keras.utils.to_categorical(y_test, num_classes=num_classes)

print(f"Training data shape: {X_train.shape}")
print(f"Test data shape: {X_test.shape}")

### Import necessary libraries for numerical operations, deep learning, image processing, and plotting import numpy as np import tensorflow as tf import cv2 import os import matplotlib.pyplot as plt  ### Define paths to the training and testing datasets trainPath = "C:/Data-Sets/Emotion-Faces/train" testPath = "C:/Data-Sets/Emotion-Faces/test"  ### Get the list of emotion categories from the training folder folderList = sorted(os.listdir(trainPath)) print(folderList)  ### Initialize empty arrays to store training and testing images and labels X_train, y_train = [], [] X_test, y_test = [], []  ### Loop through training data and load each image in grayscale format for i, category in enumerate(folderList):     files = os.listdir(os.path.join(trainPath, category))     for file in files:         print(f"{category}/{file}")         img = cv2.imread(os.path.join(trainPath, category, file), cv2.IMREAD_GRAYSCALE)         X_train.append(img)         y_train.append(i)  # Assign numeric label  print(f"Number of training images: {len(X_train)}")  ### Load testing data in the same way as training data folderList = sorted(os.listdir(testPath)) for i, category in enumerate(folderList):     files = os.listdir(os.path.join(testPath, category))     for file in files:         print(f"{category}/{file}")         img = cv2.imread(os.path.join(testPath, category, file), cv2.IMREAD_GRAYSCALE)         X_test.append(img)         y_test.append(i)  print(f"Number of test images: {len(X_test)}")  ### Convert lists to numpy arrays for better performance X_train = np.array(X_train, dtype='float32') y_train = np.array(y_train, dtype='float32') X_test = np.array(X_test, dtype='float32') y_test = np.array(y_test, dtype='float32')  ### Normalize image pixel values to the range 0-1 X_train /= 255.0 X_test /= 255.0  ### Reshape images to include channel dimension (48x48x1) X_train = X_train.reshape(-1, 48, 48, 1) X_test = X_test.reshape(-1, 48, 48, 1)  ### Convert emotion labels into categorical format (7 classes) num_classes = 7 y_train = tf.keras.utils.to_categorical(y_train, num_classes=num_classes) y_test = tf.keras.utils.to_categorical(y_test, num_classes=num_classes)  print(f"Training data shape: {X_train.shape}") print(f"Test data shape: {X_test.shape}") 

### Define a CNN architecture for emotion detection
model = tf.keras.Sequential([
    tf.keras.layers.Conv2D(64, (3, 3), padding="same", activation="relu", input_shape=(48, 48, 1)),
    tf.keras.layers.Conv2D(64, (3, 3), padding="same", activation="relu"),
    tf.keras.layers.MaxPooling2D((2, 2), strides=(2, 2)),

    tf.keras.layers.Conv2D(128, (3, 3), padding="same", activation="relu"),
    tf.keras.layers.Conv2D(128, (3, 3), padding="same", activation="relu"),
    tf.keras.layers.MaxPooling2D((2, 2), strides=(2, 2)),

    tf.keras.layers.Conv2D(256, (3, 3), padding="same", activation="relu"),
    tf.keras.layers.Conv2D(256, (3, 3), padding="same", activation="relu"),
    tf.keras.layers.Conv2D(256, (3, 3), padding="same", activation="relu"),
    tf.keras.layers.MaxPooling2D((2, 2), strides=(2, 2)),

    tf.keras.layers.Conv2D(512, (3, 3), padding="same", activation="relu"),
    tf.keras.layers.Conv2D(512, (3, 3), padding="same", activation="relu"),
    tf.keras.layers.Conv2D(512, (3, 3), padding="same", activation="relu"),
    tf.keras.layers.MaxPooling2D((2, 2), strides=(2, 2)),

    tf.keras.layers.Conv2D(512, (3, 3), padding="same", activation="relu"),
    tf.keras.layers.Conv2D(512, (3, 3), padding="same", activation="relu"),
    tf.keras.layers.Conv2D(512, (3, 3), padding="same", activation="relu"),
    tf.keras.layers.MaxPooling2D((2, 2), strides=(2, 2)),

    tf.keras.layers.Flatten(),
    tf.keras.layers.Dense(4096, activation="relu"),
    tf.keras.layers.Dropout(0.5),
    tf.keras.layers.Dense(4096, activation="relu"),
    tf.keras.layers.Dense(num_classes, activation="softmax")
])

### Compile the CNN model with Adam optimizer and categorical crossentropy loss
model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001),
              loss='categorical_crossentropy',
              metrics=['accuracy'])

print(model.summary())

### Define training parameters
batch_size = 32
epochs = 30

### Use early stopping to prevent overfitting
early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=5, restore_best_weights=True)

### Train the model on training data and validate on test data
history = model.fit(
    X_train, y_train,
    batch_size=batch_size,
    epochs=epochs,
    validation_data=(X_test, y_test),
    shuffle=True,
    callbacks=[early_stopping]
)

### Define a CNN architecture for emotion detection model = tf.keras.Sequential([     tf.keras.layers.Conv2D(64, (3, 3), padding="same", activation="relu", input_shape=(48, 48, 1)),     tf.keras.layers.Conv2D(64, (3, 3), padding="same", activation="relu"),     tf.keras.layers.MaxPooling2D((2, 2), strides=(2, 2)),      tf.keras.layers.Conv2D(128, (3, 3), padding="same", activation="relu"),     tf.keras.layers.Conv2D(128, (3, 3), padding="same", activation="relu"),     tf.keras.layers.MaxPooling2D((2, 2), strides=(2, 2)),      tf.keras.layers.Conv2D(256, (3, 3), padding="same", activation="relu"),     tf.keras.layers.Conv2D(256, (3, 3), padding="same", activation="relu"),     tf.keras.layers.Conv2D(256, (3, 3), padding="same", activation="relu"),     tf.keras.layers.MaxPooling2D((2, 2), strides=(2, 2)),      tf.keras.layers.Conv2D(512, (3, 3), padding="same", activation="relu"),     tf.keras.layers.Conv2D(512, (3, 3), padding="same", activation="relu"),     tf.keras.layers.Conv2D(512, (3, 3), padding="same", activation="relu"),     tf.keras.layers.MaxPooling2D((2, 2), strides=(2, 2)),      tf.keras.layers.Conv2D(512, (3, 3), padding="same", activation="relu"),     tf.keras.layers.Conv2D(512, (3, 3), padding="same", activation="relu"),     tf.keras.layers.Conv2D(512, (3, 3), padding="same", activation="relu"),     tf.keras.layers.MaxPooling2D((2, 2), strides=(2, 2)),      tf.keras.layers.Flatten(),     tf.keras.layers.Dense(4096, activation="relu"),     tf.keras.layers.Dropout(0.5),     tf.keras.layers.Dense(4096, activation="relu"),     tf.keras.layers.Dense(num_classes, activation="softmax") ])  ### Compile the CNN model with Adam optimizer and categorical crossentropy loss model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001),               loss='categorical_crossentropy',               metrics=['accuracy'])  print(model.summary())  ### Define training parameters batch_size = 32 epochs = 30  ### Use early stopping to prevent overfitting early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=5, restore_best_weights=True)  ### Train the model on training data and validate on test data history = model.fit(     X_train, y_train,     batch_size=batch_size,     epochs=epochs,     validation_data=(X_test, y_test),     shuffle=True,     callbacks=[early_stopping] ) 

### Extract accuracy and loss from training history
acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']

### Plot training vs validation accuracy
plt.figure()
plt.plot(acc, label='Training Accuracy', color='r')
plt.plot(val_acc, label='Validation Accuracy', color='b')
plt.title('Training and Validation Accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()

### Plot training vs validation loss
plt.figure()
plt.plot(loss, label='Training Loss', color='r')
plt.plot(val_loss, label='Validation Loss', color='b')
plt.title('Training and Validation Loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()
plt.show()

### Save the trained model for later use
model.save("e:/temp/emotion_model.h5")

### Extract accuracy and loss from training history acc = history.history['accuracy'] val_acc = history.history['val_accuracy'] loss = history.history['loss'] val_loss = history.history['val_loss']  ### Plot training vs validation accuracy plt.figure() plt.plot(acc, label='Training Accuracy', color='r') plt.plot(val_acc, label='Validation Accuracy', color='b') plt.title('Training and Validation Accuracy') plt.xlabel('Epochs') plt.ylabel('Accuracy') plt.legend()  ### Plot training vs validation loss plt.figure() plt.plot(loss, label='Training Loss', color='r') plt.plot(val_loss, label='Validation Loss', color='b') plt.title('Training and Validation Loss') plt.xlabel('Epochs') plt.ylabel('Loss') plt.legend() plt.show()  ### Save the trained model for later use model.save("e:/temp/emotion_model.h5") 

### Import TensorFlow (Keras backend) to load and run the trained CNN for emotion detection.
import tensorflow as tf
### Import Keras image helpers to support optional array conversion if needed later.
from keras.utils import img_to_array , load_img
### Import NumPy for numerical operations on arrays and model outputs.
import numpy as np
### Import OpenCV for computer vision tasks like face detection and image preprocessing.
import cv2
### Import os to work with filesystem paths and directory listings.
import os

### Define the path to the saved Keras model that performs facial emotion recognition.
model_file = "e:/temp/emotion.h5"
### Load the pretrained model from disk so it can be used for inference.
model = tf.keras.models.load_model(model_file)
### Print a model summary to verify the architecture and ensure successful loading.
print(model.summary())

### Set a default batch size (not required for single image, but handy for consistency).
batchSize = 32

### Informative print to mark the start of category discovery.
print("Categories :")
### Path of the training set used originally to infer the folder names (emotion classes).
trainPath = "C:/Data-Sets/Emotion-Faces/train"
### Read the immediate subfolders—each folder name represents an emotion class.
categories = os.listdir(trainPath)
### Sort categories to ensure consistent class-index mapping.
categories.sort()
### Print the sorted class list to verify labels order.
print(categories)
### Count how many emotion classes exist (e.g., 7 for FER2013).
numOfClasses = len(categories)
### Print the number of classes to confirm expectations.
print(numOfClasses)

### Define a function to locate a face in a given image path using OpenCV Haar Cascades.
def findFace(pathForImage):
    ### Read the image from disk in BGR color format (OpenCV default).
    image = cv2.imread(pathForImage)
    ### Convert the image to grayscale because Haar Cascades expect single-channel input.
    gray = cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)

    ### Path to the Haar Cascade XML (download the file to this location beforehand).
    haarCascadeFile="e:/temp/haarcascade_frontalface_default.xml"
    ### Load the cascade classifier into memory for face detection.
    face_cascade = cv2.CascadeClassifier(haarCascadeFile)
    ### Detect faces; returns a list of bounding boxes (x, y, w, h).
    faces = face_cascade.detectMultiScale(gray)

    ### Iterate over detected faces and extract the first face ROI as grayscale.
    for (x,y,w,h) in faces :
        ### Crop the region of interest from the grayscale frame.
        roiGray = gray[y:y+h , x:x+w]
        
    ### Return the grayscale face ROI (assumes at least one face found).
    return roiGray
    ### Optional visualization lines are commented out for headless/server runs.
    #cv2.imshow("img",gray)
    #cv2.waitKey(0)

### Define a helper function to resize and scale the face ROI for the CNN input.
def prepareImageForModel(faceImage):
    ### Resize the cropped face to the expected 48x48 resolution with area interpolation.
    resized = cv2.resize(faceImage, (48,48), interpolation=cv2.INTER_AREA)
    ### Add a batch dimension (1, 48, 48) so the model can process the image.
    imgResult = np.expand_dims(resized, axis=0)
    ### Scale pixel values to [0, 1] to match model training normalization.
    imgResult = imgResult / 255.0
    ### Return the preprocessed tensor ready for emotion recognition.
    return imgResult

### Import TensorFlow (Keras backend) to load and run the trained CNN for emotion detection. import tensorflow as tf ### Import Keras image helpers to support optional array conversion if needed later. from keras.utils import img_to_array , load_img ### Import NumPy for numerical operations on arrays and model outputs. import numpy as np ### Import OpenCV for computer vision tasks like face detection and image preprocessing. import cv2 ### Import os to work with filesystem paths and directory listings. import os  ### Define the path to the saved Keras model that performs facial emotion recognition. model_file = "e:/temp/emotion.h5" ### Load the pretrained model from disk so it can be used for inference. model = tf.keras.models.load_model(model_file) ### Print a model summary to verify the architecture and ensure successful loading. print(model.summary())  ### Set a default batch size (not required for single image, but handy for consistency). batchSize = 32  ### Informative print to mark the start of category discovery. print("Categories :") ### Path of the training set used originally to infer the folder names (emotion classes). trainPath = "C:/Data-Sets/Emotion-Faces/train" ### Read the immediate subfolders—each folder name represents an emotion class. categories = os.listdir(trainPath) ### Sort categories to ensure consistent class-index mapping. categories.sort() ### Print the sorted class list to verify labels order. print(categories) ### Count how many emotion classes exist (e.g., 7 for FER2013). numOfClasses = len(categories) ### Print the number of classes to confirm expectations. print(numOfClasses)  ### Define a function to locate a face in a given image path using OpenCV Haar Cascades. def findFace(pathForImage):     ### Read the image from disk in BGR color format (OpenCV default).     image = cv2.imread(pathForImage)     ### Convert the image to grayscale because Haar Cascades expect single-channel input.     gray = cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)      ### Path to the Haar Cascade XML (download the file to this location beforehand).     haarCascadeFile="e:/temp/haarcascade_frontalface_default.xml"     ### Load the cascade classifier into memory for face detection.     face_cascade = cv2.CascadeClassifier(haarCascadeFile)     ### Detect faces; returns a list of bounding boxes (x, y, w, h).     faces = face_cascade.detectMultiScale(gray)      ### Iterate over detected faces and extract the first face ROI as grayscale.     for (x,y,w,h) in faces :         ### Crop the region of interest from the grayscale frame.         roiGray = gray[y:y+h , x:x+w]              ### Return the grayscale face ROI (assumes at least one face found).     return roiGray     ### Optional visualization lines are commented out for headless/server runs.     #cv2.imshow("img",gray)     #cv2.waitKey(0)  ### Define a helper function to resize and scale the face ROI for the CNN input. def prepareImageForModel(faceImage):     ### Resize the cropped face to the expected 48x48 resolution with area interpolation.     resized = cv2.resize(faceImage, (48,48), interpolation=cv2.INTER_AREA)     ### Add a batch dimension (1, 48, 48) so the model can process the image.     imgResult = np.expand_dims(resized, axis=0)     ### Scale pixel values to [0, 1] to match model training normalization.     imgResult = imgResult / 255.0     ### Return the preprocessed tensor ready for emotion recognition.     return imgResult 

### Provide the path to a test image that contains a human face for emotion detection.
testImagePath = "TensorFlowProjects\\Emotion-Detection\\suprised.jpg"
### Alternative test image path for another expression (commented out).
#testImagePath = "TensorFlowProjects\\Emotion-Detection\\Happy.jpg"

### Call the face finder to extract a grayscale face ROI from the image.
faceGrayImage = findFace(testImagePath)

### Preprocess the face ROI to match the model’s input format (48x48 normalized).
imgForModel = prepareImageForModel(faceGrayImage)

### Run the CNN model to obtain the probability distribution over emotion classes.
resultArray = model.predict(imgForModel, verbose=1)
### Convert softmax probabilities to the index of the highest-scoring class.
answers = np.argmax(resultArray, axis=1)

### Print the numeric class index for quick debugging.
print(answers[0])

### Map the predicted index back to its human-readable label from the categories list.
text = categories[answers[0]]

### Display the predicted label in the console for verification.
print("Predicted : " + text)

### Read the original image again for drawing the overlay text.
img = cv2.imread(testImagePath)
### Choose a readable OpenCV font for the overlay.
font = cv2.FONT_HERSHEY_COMPLEX

### Put the predicted emotion text at the top-left corner of the image.
cv2.putText(img, text, (0,20), font, 0.5, (209,19,77), 2)
### Show the resulting image with the predicted label overlay.
cv2.imshow("img",img)

### Wait for a key press so the image window remains visible.
cv2.waitKey(0)

### Close any OpenCV windows after the visualization is done.
cv2.destroyAllWindows()

### Example: expected class count and label order comment kept for reference.
#6
#['angry', 'disgust', 'fear', 'happy', 'neutral', 'sad', 'surprise']

### Optional: display the cropped grayscale face ROI (useful for debugging).
#cv2.imshow("img",faceGrayImage)
#cv2.waitKey(0)

### Provide the path to a test image that contains a human face for emotion detection. testImagePath = "TensorFlowProjects\\Emotion-Detection\\suprised.jpg" ### Alternative test image path for another expression (commented out). #testImagePath = "TensorFlowProjects\\Emotion-Detection\\Happy.jpg"  ### Call the face finder to extract a grayscale face ROI from the image. faceGrayImage = findFace(testImagePath)  ### Preprocess the face ROI to match the model’s input format (48x48 normalized). imgForModel = prepareImageForModel(faceGrayImage)  ### Run the CNN model to obtain the probability distribution over emotion classes. resultArray = model.predict(imgForModel, verbose=1) ### Convert softmax probabilities to the index of the highest-scoring class. answers = np.argmax(resultArray, axis=1)  ### Print the numeric class index for quick debugging. print(answers[0])  ### Map the predicted index back to its human-readable label from the categories list. text = categories[answers[0]]  ### Display the predicted label in the console for verification. print("Predicted : " + text)  ### Read the original image again for drawing the overlay text. img = cv2.imread(testImagePath) ### Choose a readable OpenCV font for the overlay. font = cv2.FONT_HERSHEY_COMPLEX  ### Put the predicted emotion text at the top-left corner of the image. cv2.putText(img, text, (0,20), font, 0.5, (209,19,77), 2) ### Show the resulting image with the predicted label overlay. cv2.imshow("img",img)  ### Wait for a key press so the image window remains visible. cv2.waitKey(0)  ### Close any OpenCV windows after the visualization is done. cv2.destroyAllWindows()  ### Example: expected class count and label order comment kept for reference. #6 #['angry', 'disgust', 'fear', 'happy', 'neutral', 'sad', 'surprise']  ### Optional: display the cropped grayscale face ROI (useful for debugging). #cv2.imshow("img",faceGrayImage) #cv2.waitKey(0)