The metric has not yet been built error

Hi everyone,

i was stacking 5 pretrain model sort of like transfer learning.
I was facing issue about The metric has not yet been built error issue and I not sure how to solve it.

i am using tensorflow 2.17 version. I had check all model in built status and also the metric are correct. I also ensure all model input and output shape are compatibility.

Below is my code and error message.

from sklearn.datasets import make_blobs
from sklearn.metrics import accuracy_score
import tensorflow as tf
from tensorflow.keras.utils import plot_model
from tensorflow.keras.models import Model, load_model
from tensorflow.keras.layers import Input, Dense, Concatenate
from tensorflow.keras.utils import to_categorical
from numpy import argmax
import IPython

print(tf.__version__)


def define_stacked_model():
    ensemble_visible = []
    ensemble_outputs = []
    for i in range(5):
        filename = 'models/model_' + str(i + 1) + '.keras'
        model = load_model(filename)
        for layer in model.layers:
            layer.trainable = False
        input = Input(shape=(model.input_shape[-1],), name=f'ensemble_input_{i+1}')
        ensemble_visible.append(input)
        output = model(input)
        ensemble_outputs.append(output)

    merge = Concatenate(axis=-1, name='merge_layer')(ensemble_outputs)
    hidden = Dense(10, activation='relu', name='hidden_layer')(merge)
    output = Dense(3, activation='softmax', name='output_layer')(hidden)
    
    final_model = Model(inputs=ensemble_visible, outputs=output, name='stacked_model')
    final_model.build(input_shape=ensemble_visible)
    final_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['categorical_accuracy'])
    plot_model(final_model, to_file='stacked_model_plot.png', show_shapes=True, show_layer_names=True, expand_nested=True)
    IPython.display.display(IPython.display.Image("stacked_model_plot.png"))
    return final_model

stacked_model = define_stacked_model()

def fit_stacked_model(model, inputX, inputy):
  X = [inputX for _ in range(len(model.input_shape))]
  inputy_enc = to_categorical(inputy)
  model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['categorical_accuracy'])
  model.fit(X, inputy_enc, epochs=300, verbose=0)

def predict_stacked_model(model, inputX):
    X = [inputX for _ in range(len(model.input_shape))]
    return model.predict(X, verbose=0)

X, y = make_blobs(n_samples=1100, centers=3, n_features=2, cluster_std=2, random_state=2)
n_train = 100
trainX, testX = X[:n_train, :], X[n_train:, :]
trainy, testy = y[:n_train], y[n_train:]

fit_stacked_model(stacked_model, trainX, trainy)
yhat = predict_stacked_model(stacked_model, testX)
yhat = argmax(yhat, axis=1)
acc = accuracy_score(testy, yhat)
print('Stacked Test Accuracy: %.3f' % acc)
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[37], line 55
     52 trainX, testX = X[:n_train, :], X[n_train:, :]
     53 trainy, testy = y[:n_train], y[n_train:]
---> 55 fit_stacked_model(stacked_model, trainX, trainy)
     56 yhat = predict_stacked_model(stacked_model, testX)
     57 yhat = argmax(yhat, axis=1)

Cell In[37], line 44, in fit_stacked_model(model, inputX, inputy)
     42 inputy_enc = to_categorical(inputy)
     43 model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['categorical_accuracy'])
---> 44 model.fit(X, inputy_enc, epochs=300, verbose=0)

File ~\Anaconda\envs\pytorch-gpu\Lib\site-packages\keras\src\utils\traceback_utils.py:122, in filter_traceback.<locals>.error_handler(*args, **kwargs)
    119     filtered_tb = _process_traceback_frames(e.__traceback__)
    120     # To get the full stack trace, call:
    121     # `keras.config.disable_traceback_filtering()`
--> 122     raise e.with_traceback(filtered_tb) from None
    123 finally:
    124     del filtered_tb

File ~\Anaconda\envs\pytorch-gpu\Lib\site-packages\keras\src\trainers\compile_utils.py:356, in CompileMetrics.result(self)
    354 def result(self):
    355     if not self.built:
--> 356         raise ValueError(
    357             "Cannot get result() since the metric has not yet been built."
    358         )
    359     results = {}
    360     unique_name_counters = {}

ValueError: Cannot get result() since the metric has not yet been built.

Hi @Clarence_Ng, If possible could you please provide the model files which you are trying to load in order to reproduce the issue. Thank You.

Hi Kikran,
Below is the code for generate the model

from sklearn.datasets import make_blobs
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Input
from matplotlib import pyplot
from os import makedirs
import os.path
import tensorflow as tf


# fit model on dataset
def fit_model(trainX, trainy):
    # define model
    model = Sequential()
    model.add(Input(shape=(2,)))  # Use Input layer here
    model.add(Dense(25, activation='relu'))
    model.add(Dense(3, activation='softmax'))
    
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    # fit model
    model.fit(trainX, trainy, epochs=500, verbose=0)
    return model
 
# generate 2d classification dataset
X, y = make_blobs(n_samples=1100, centers=3, n_features=2, cluster_std=2, random_state=2)
# one hot encode output variable
y = to_categorical(y)
# split into train and test
n_train = 100
trainX, testX = X[:n_train, :], X[n_train:, :]
trainy, testy = y[:n_train], y[n_train:]
print(trainX.shape, testX.shape)
# create directory for models
#makedirs('models')
if not os.path.exists('models'):
    os.makedirs('models')

# fit and save models
n_members = 5
for i in range(n_members):
    # fit model
    model = fit_model(trainX, trainy)
    # save model
    filename = 'models/model_' + str(i + 1) + '.keras'

    model.save(filename)
    print('>Saved %s' % filename)