Hi Kiran_Sai_Ramineni, Thank you very much for your detailed response. The input data is 12-channel ECG, and here’s the full code, I still get different results using inbuilt and custom loss functions.
from numpy import mean
from numpy import std
from matplotlib import pyplot as plt
from sklearn.model_selection import KFold
from keras.datasets import mnist
from tensorflow.keras.utils import to_categorical
from keras.models import Sequential
from keras.layers import Conv2D
from keras.layers import MaxPooling2D
from keras.layers import Dense
from keras.layers import Flatten
#from keras.utils import plot_model
from keras.models import Model
from keras.layers import Input
from keras.layers import concatenate
from tensorflow.keras.optimizers import SGD
from keras.callbacks import EarlyStopping
from keras.callbacks import ModelCheckpoint
import numpy as np
from keras.models import load_model
import scipy.io as sio
import numpy as np
from keras import optimizers
import keras.backend as K
from sklearn.metrics import roc_curve
import tensorflow as tf
from sklearn import metrics
import hdf5storage
import keras
import keras_tuner
from keras_tuner import HyperModel
from keras_tuner.tuners import BayesianOptimization
from keras_tuner.tuners import RandomSearch
import os
from random import seed
def bce_loss(y_true, y_pred):
epsilon = tf.keras.backend.epsilon()
y_true = tf.cast(y_true, dtype=tf.float32)
y_pred = tf.cast(y_pred, dtype=tf.float32)
epsilon_ = tf.convert_to_tensor(epsilon, dtype=y_pred.dtype)
y_pred = tf.clip_by_value(y_pred, epsilon_, 1. - epsilon_)
bce = y_true * tf.math.log(y_pred + epsilon_) + (1 - y_true) * tf.math.log(1 - y_pred + epsilon_)
loss = -tf.reduce_mean( bce )
return loss
for foldInd in range(1,2):
seed(1)
foldData = r'ECG.mat'
TrainTestVal_data = hdf5storage.loadmat(foldData)
dataTrain = TrainTestVal_data['Train_ECG']
labelsTrain = TrainTestVal_data['TrainLabel']
dataTrain = TrainTestVal_data['Train_ECG']
labelsTrain = TrainTestVal_data['TrainLabel']
indices = np.arange(dataTrain.shape[0])
np.random.shuffle(indices)
shuffled_train_data = dataTrain[indices]
shuffled_train_labels = labelsTrain[indices]
# trainY = tf.keras.utils.to_categorical( shuffled_train_labels, num_classes=2)
trainY = shuffled_train_labels
dataTest = TrainTestVal_data['Test_ECG']
labelsTest = TrainTestVal_data['TestLabel']
indices = np.arange(dataTest.shape[0])
np.random.shuffle(indices)
shuffled_test_data = dataTest[indices]
shuffled_test_labels = labelsTest[indices]
# testY = tf.keras.utils.to_categorical(shuffled_test_labels, num_classes=2)
testY = shuffled_test_labels
dataVal = TrainTestVal_data['Valid_ECG']
labelsVal = TrainTestVal_data['ValidLabel']
indices = np.arange(dataVal.shape[0])
np.random.shuffle(indices)
shuffled_valid_data = dataVal[indices]
shuffled_valid_labels = labelsVal[indices]
# valY = tf.keras.utils.to_categorical(shuffled_valid_labels, num_classes=2)
valY=shuffled_valid_labels
trainECG = shuffled_train_data
trainECG1 = trainECG[:,0:1,:]
trainECG2 = trainECG[:,1:2,:]
trainECG3 = trainECG[:,2:3,:]
trainECG4 = trainECG[:,3:4,:]
trainECG5 = trainECG[:,4:5,:]
trainECG6 = trainECG[:,5:6,:]
trainECG7 = trainECG[:,6:7,:]
trainECG8 = trainECG[:,7:8,:]
trainECG9 = trainECG[:,8:9,:]
trainECG10 = trainECG[:,9:10,:]
trainECG11 = trainECG[:,10:11,:]
trainECG12 = trainECG[:,11:12,:]
testECG = shuffled_test_data
testECG1 = testECG[:,0:1,:]
testECG2 = testECG[:,1:2,:]
testECG3 = testECG[:,2:3,:]
testECG4 = testECG[:,3:4,:]
testECG5 = testECG[:,4:5,:]
testECG6 = testECG[:,5:6,:]
testECG7 = testECG[:,6:7,:]
testECG8 = testECG[:,7:8,:]
testECG9 = testECG[:,8:9,:]
testECG10 = testECG[:,9:10,:]
testECG11 = testECG[:,10:11,:]
testECG12 = testECG[:,11:12,:]
valECG = shuffled_valid_data
valECG1 = valECG[:,0:1,:]
valECG2 = valECG[:,1:2,:]
valECG3 = valECG[:,2:3,:]
valECG4 = valECG[:,3:4,:]
valECG5 = valECG[:,4:5,:]
valECG6 = valECG[:,5:6,:]
valECG7 = valECG[:,6:7,:]
valECG8 = valECG[:,7:8,:]
valECG9 = valECG[:,8:9,:]
valECG10 = valECG[:,9:10,:]
valECG11 = valECG[:,10:11,:]
valECG12 = valECG[:,11:12,:]
RawInputECG1 = Input(shape=(1,301,1))
RawInputECG1 = Input(shape=(1,301,1))
RawInputECG2 = Input(shape=(1,301,1))
RawInputECG3 = Input(shape=(1,301,1))
RawInputECG4 = Input(shape=(1,301,1))
RawInputECG5 = Input(shape=(1,301,1))
RawInputECG6 = Input(shape=(1,301,1))
RawInputECG7 = Input(shape=(1,301,1))
RawInputECG8 = Input(shape=(1,301,1))
RawInputECG9 = Input(shape=(1,301,1))
RawInputECG10 = Input(shape=(1,301,1))
RawInputECG11 = Input(shape=(1,301,1))
RawInputECG12 = Input(shape=(1,301,1))
convECG1 = Conv2D(filters=8, kernel_size=(1,50), activation='sigmoid')(RawInputECG1)
convECG2 = Conv2D(filters=8, kernel_size= (1,50), activation='sigmoid')(RawInputECG2)
convECG3 = Conv2D(filters=8, kernel_size= (1,50), activation='sigmoid')(RawInputECG3)
convECG4 = Conv2D(filters=8, kernel_size= (1,50), activation='sigmoid')(RawInputECG4)
convECG5 = Conv2D(filters=8, kernel_size= (1,50), activation='sigmoid')(RawInputECG5)
convECG6 = Conv2D(filters=8, kernel_size= (1,50), activation='sigmoid')(RawInputECG6)
convECG7 = Conv2D(filters=8, kernel_size= (1,50), activation='sigmoid')(RawInputECG7)
convECG8 = Conv2D(filters=8, kernel_size= (1,50), activation='sigmoid')(RawInputECG8)
convECG9 = Conv2D(filters=8, kernel_size= (1,50), activation='sigmoid')(RawInputECG9)
convECG10 = Conv2D(filters=8, kernel_size= (1,50), activation='sigmoid')(RawInputECG10)
convECG11 = Conv2D(filters=8, kernel_size= (1,50), activation='sigmoid')(RawInputECG11)
convECG12 = Conv2D(filters=8, kernel_size= (1,50), activation='sigmoid')(RawInputECG12)
# merge1 = concatenate([f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12], axis = 1)
# f1 = Flatten()(convECG12)
# flat = Flatten()(f1)
f1 = Flatten()(convECG1)
f2 = Flatten()(convECG2)
f3 = Flatten()(convECG3)
f4 = Flatten()(convECG4)
f5 = Flatten()(convECG5)
f6 = Flatten()(convECG6)
f7 = Flatten()(convECG7)
f8 = Flatten()(convECG8)
f9 = Flatten()(convECG9)
f10 = Flatten()(convECG10)
f11 = Flatten()(convECG11)
f12 = Flatten()(convECG12)
# Merge all the flattened outputs
merge1 = concatenate([f1, f2, f3, f4, f5, f6,f7,f8,f9,f10,f11,f12], axis=1)
# Output layer
output = Dense(1, activation='sigmoid')(merge1)
model = Model(inputs=[RawInputECG1, RawInputECG2, RawInputECG3, RawInputECG4, RawInputECG5, RawInputECG6,
RawInputECG7, RawInputECG8, RawInputECG9, RawInputECG10, RawInputECG11, RawInputECG12],
outputs=output)
Calling the custom loss:
model.compile(optimizer=optim, loss=bce_loss, metrics=['accuracy', auc])
saveFN = 'bce_loss_ECG.h5'
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=20)
mc = ModelCheckpoint(saveFN, monitor='val_loss', mode='min', verbose=1, save_best_only=True)
# Train the model with debugging enabled
history = model.fit([trainECG1, trainECG2, trainECG3, trainECG4, trainECG5, trainECG6, trainECG7, trainECG8, trainECG9, trainECG10, trainECG11, trainECG12], trainY, epochs=70, batch_size=32, verbose=1,
validation_data=([valECG1, valECG2, valECG3, valECG4, valECG5, valECG6, valECG7, valECG8, valECG9, valECG10, valECG11, valECG12], valY), callbacks=[es,mc]
)
Here’s the output:
Epoch 18/70
484/487 [============================>.] - ETA: 0s - loss: 7.6226 - accuracy: 0.5001 - auc: 0.5000
Epoch 18: val_loss did not improve from 7.55872
487/487 [==============================] - 7s 14ms/step - loss: 7.6251 - accuracy: 0.5000 - auc: 0.5000 - val_loss: 7.5587 - val_accuracy: 0.5043 - val_auc: 0.5000
Epoch 19/70
483/487 [============================>.] - ETA: 0s - loss: 7.6246 - accuracy: 0.5000 - auc: 0.5000
Epoch 19: val_loss did not improve from 7.55872
487/487 [==============================] - 7s 15ms/step - loss: 7.6251 - accuracy: 0.5000 - auc: 0.5000 - val_loss: 7.5587 - val_accuracy: 0.5043 - val_auc: 0.5000
Epoch 20/70
483/487 [============================>.] - ETA: 0s - loss: 7.6325 - accuracy: 0.4995 - auc: 0.5000
Epoch 20: val_loss did not improve from 7.55872
487/487 [==============================] - 6s 13ms/step - loss: 7.6251 - accuracy: 0.5000 - auc: 0.5000 - val_loss: 7.5587 - val_accuracy: 0.5043 - val_auc: 0.5000
Epoch 21/70
485/487 [============================>.] - ETA: 0s - loss: 7.6266 - accuracy: 0.4999 - auc: 0.5000
Epoch 21: val_loss did not improve from 7.55872
487/487 [==============================] - 6s 13ms/step - loss: 7.6251 - accuracy: 0.5000 - auc: 0.5000 - val_loss: 7.5587 - val_accuracy: 0.5043 - val_auc: 0.5000
Epoch 21: early stopping
Calling the inbuilt loss function:
# Compile model with eager execution enabled
optim = tf.keras.optimizers.SGD(learning_rate=0.01, momentum=0.0, nesterov=False)
auc = tf.keras.metrics.AUC()
model.compile(optimizer=optim, loss='binary_crossentropy', metrics=['accuracy', auc])
saveFN = 'bce_loss_ECG.h5'
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=20)
mc = ModelCheckpoint(saveFN, monitor='val_loss', mode='min', verbose=1, save_best_only=True)
# Train the model with debugging enabled
history = model.fit([trainECG1, trainECG2, trainECG3, trainECG4, trainECG5, trainECG6, trainECG7, trainECG8, trainECG9, trainECG10, trainECG11, trainECG12], trainY, epochs=70, batch_size=32, verbose=1,
validation_data=([valECG1, valECG2, valECG3, valECG4, valECG5, valECG6, valECG7, valECG8, valECG9, valECG10, valECG11, valECG12], valY), callbacks=[es,mc]
)
Here’s the output:
Epoch 68/70
487/487 [==============================] - ETA: 0s - loss: 0.7145 - accuracy: 0.5350 - auc: 0.5532
Epoch 68: val_loss did not improve from 0.69349
487/487 [==============================] - 6s 13ms/step - loss: 0.7145 - accuracy: 0.5350 - auc: 0.5532 - val_loss: 0.7261 - val_accuracy: 0.5094 - val_auc: 0.5525
Epoch 69/70
484/487 [============================>.] - ETA: 0s - loss: 0.7246 - accuracy: 0.5316 - auc: 0.5436
Epoch 69: val_loss did not improve from 0.69349
487/487 [==============================] - 6s 13ms/step - loss: 0.7247 - accuracy: 0.5313 - auc: 0.5433 - val_loss: 0.7417 - val_accuracy: 0.5150 - val_auc: 0.5533
Epoch 70/70
486/487 [============================>.] - ETA: 0s - loss: 0.7160 - accuracy: 0.5348 - auc: 0.5526
Epoch 70: val_loss did not improve from 0.69349
487/487 [==============================] - 7s 13ms/step - loss: 0.7159 - accuracy: 0.5350 - auc: 0.5528 - val_loss: 0.7004 - val_accuracy: 0.5440 - val_auc: 0.5540
Regarding the software: I have created a virtual environment called ‘tf-env’ and installed the tensorflow and Keras version : 2.13.0.
Could you please confirm if this is due to issues with the virtual environment?
update:
I have tried running this code in Google Colab and found similar results for the bce_loss function (custom function). The tensorflow version is: 2.17.1
Epoch 20: val_loss did not improve from 7.55872
487/487 ━━━━━━━━━━━━━━━━━━━━ 34s 71ms/step - accuracy: 0.5038 - auc_1: 0.5000 - loss: 7.5671 - val_accuracy: 0.5043 - val_auc_1: 0.5000 - val_loss: 7.5587
Epoch 21/70
486/487 ━━━━━━━━━━━━━━━━━━━━ 0s 69ms/step - accuracy: 0.4978 - auc_1: 0.5000 - loss: 7.6578
Epoch 21: val_loss did not improve from 7.55872
487/487 ━━━━━━━━━━━━━━━━━━━━ 41s 71ms/step - accuracy: 0.4978 - auc_1: 0.5000 - loss: 7.6577 - val_accuracy: 0.5043 - val_auc_1: 0.5000 - val_loss: 7.5587
Epoch 21: early stopping
Thank you very much
Geerthy