The metric has not yet been built error

Keras
,
1

Hi everyone,

i was stacking 5 pretrain model sort of like transfer learning.
I was facing issue about The metric has not yet been built error issue and I not sure how to solve it.

i am using tensorflow 2.17 version. I had check all model in built status and also the metric are correct. I also ensure all model input and output shape are compatibility.

Below is my code and error message.

from sklearn.datasets import make_blobs
from sklearn.metrics import accuracy_score
import tensorflow as tf
from tensorflow.keras.utils import plot_model
from tensorflow.keras.models import Model, load_model
from tensorflow.keras.layers import Input, Dense, Concatenate
from tensorflow.keras.utils import to_categorical
from numpy import argmax
import IPython

print(tf.__version__)


def define_stacked_model():
    ensemble_visible = []
    ensemble_outputs = []
    for i in range(5):
        filename = 'models/model_' + str(i + 1) + '.keras'
        model = load_model(filename)
        for layer in model.layers:
            layer.trainable = False
        input = Input(shape=(model.input_shape[-1],), name=f'ensemble_input_{i+1}')
        ensemble_visible.append(input)
        output = model(input)
        ensemble_outputs.append(output)

    merge = Concatenate(axis=-1, name='merge_layer')(ensemble_outputs)
    hidden = Dense(10, activation='relu', name='hidden_layer')(merge)
    output = Dense(3, activation='softmax', name='output_layer')(hidden)
    
    final_model = Model(inputs=ensemble_visible, outputs=output, name='stacked_model')
    final_model.build(input_shape=ensemble_visible)
    final_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['categorical_accuracy'])
    plot_model(final_model, to_file='stacked_model_plot.png', show_shapes=True, show_layer_names=True, expand_nested=True)
    IPython.display.display(IPython.display.Image("stacked_model_plot.png"))
    return final_model

stacked_model = define_stacked_model()

def fit_stacked_model(model, inputX, inputy):
  X = [inputX for _ in range(len(model.input_shape))]
  inputy_enc = to_categorical(inputy)
  model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['categorical_accuracy'])
  model.fit(X, inputy_enc, epochs=300, verbose=0)

def predict_stacked_model(model, inputX):
    X = [inputX for _ in range(len(model.input_shape))]
    return model.predict(X, verbose=0)

X, y = make_blobs(n_samples=1100, centers=3, n_features=2, cluster_std=2, random_state=2)
n_train = 100
trainX, testX = X[:n_train, :], X[n_train:, :]
trainy, testy = y[:n_train], y[n_train:]

fit_stacked_model(stacked_model, trainX, trainy)
yhat = predict_stacked_model(stacked_model, testX)
yhat = argmax(yhat, axis=1)
acc = accuracy_score(testy, yhat)
print('Stacked Test Accuracy: %.3f' % acc)

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[37], line 55
     52 trainX, testX = X[:n_train, :], X[n_train:, :]
     53 trainy, testy = y[:n_train], y[n_train:]
---> 55 fit_stacked_model(stacked_model, trainX, trainy)
     56 yhat = predict_stacked_model(stacked_model, testX)
     57 yhat = argmax(yhat, axis=1)

Cell In[37], line 44, in fit_stacked_model(model, inputX, inputy)
     42 inputy_enc = to_categorical(inputy)
     43 model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['categorical_accuracy'])
---> 44 model.fit(X, inputy_enc, epochs=300, verbose=0)

File ~\Anaconda\envs\pytorch-gpu\Lib\site-packages\keras\src\utils\traceback_utils.py:122, in filter_traceback.<locals>.error_handler(*args, **kwargs)
    119     filtered_tb = _process_traceback_frames(e.__traceback__)
    120     # To get the full stack trace, call:
    121     # `keras.config.disable_traceback_filtering()`
--> 122     raise e.with_traceback(filtered_tb) from None
    123 finally:
    124     del filtered_tb

File ~\Anaconda\envs\pytorch-gpu\Lib\site-packages\keras\src\trainers\compile_utils.py:356, in CompileMetrics.result(self)
    354 def result(self):
    355     if not self.built:
--> 356         raise ValueError(
    357             "Cannot get result() since the metric has not yet been built."
    358         )
    359     results = {}
    360     unique_name_counters = {}

ValueError: Cannot get result() since the metric has not yet been built.
3

Hi @Clarence_Ng, If possible could you please provide the model files which you are trying to load in order to reproduce the issue. Thank You.

5

Hi Kikran,
Below is the code for generate the model

from sklearn.datasets import make_blobs
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Input
from matplotlib import pyplot
from os import makedirs
import os.path
import tensorflow as tf


# fit model on dataset
def fit_model(trainX, trainy):
    # define model
    model = Sequential()
    model.add(Input(shape=(2,)))  # Use Input layer here
    model.add(Dense(25, activation='relu'))
    model.add(Dense(3, activation='softmax'))
    
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    # fit model
    model.fit(trainX, trainy, epochs=500, verbose=0)
    return model
 
# generate 2d classification dataset
X, y = make_blobs(n_samples=1100, centers=3, n_features=2, cluster_std=2, random_state=2)
# one hot encode output variable
y = to_categorical(y)
# split into train and test
n_train = 100
trainX, testX = X[:n_train, :], X[n_train:, :]
trainy, testy = y[:n_train], y[n_train:]
print(trainX.shape, testX.shape)
# create directory for models
#makedirs('models')
if not os.path.exists('models'):
    os.makedirs('models')

# fit and save models
n_members = 5
for i in range(n_members):
    # fit model
    model = fit_model(trainX, trainy)
    # save model
    filename = 'models/model_' + str(i + 1) + '.keras'

    model.save(filename)
    print('>Saved %s' % filename)
7

Hi @Clarence_Ng, The problem has been resolved in the keras-nightly version. Please refer to this gist for working code example. Thank You.