I am having this error “AttributeError: ‘Tensor’ object has no attribute ‘nested_row_splits’” when trying to train a decoder based transformer for word prediction
this is the code
**BATCH_SIZE = 64
EPOCHS = 30
MAX_SEQUENCE_LENGTH = 394
VOCAB_SIZE = 15000
EMBED_DIM = 256
INTERMEDIATE_DIM = 512
NUM_HEADS = 8**
def process_batch(ds):
ds = tokenizer(ds)
## padd short senteces to max len using the [PAD] id
## add special tokens [START] and [END]
ds_start_end_packer = StartEndPacker(
sequence_length=MAX_SEQUENCE_LENGTH,
start_value = tokenizer.token_to_id("[START]")
)
output = ds_start_end_packer(ds)
return (output, ds)
def make_ds(seq):
dataset = tf.data.Dataset.from_tensor_slices(seq)
dataset = dataset.batch(BATCH_SIZE)
dataset = dataset.map(process_batch, num_parallel_calls=tf.data.AUTOTUNE)
dataset = dataset.shuffle(2048).prefetch(16).cache()
return dataset
Data tokenization
train_ds = make_ds(train_seq)
val_ds = make_ds(val_seq)
for x,y in train_ds.take(1):
print(f"inputs.shape: {x.shape}")
print(f"features.shape: {y.shape}")
inputs.shape: (64, 394)
features.shape: (64, None)
And the code for the transformer
decoder_inputs = Input(shape=(None,), dtype="int64", name="decoder_inputs")
x = TokenAndPositionEmbedding(
vocabulary_size= VOCAB_SIZE,
sequence_length = MAX_SEQUENCE_LENGTH,
embedding_dim = EMBED_DIM,
mask_zero =True
)(decoder_inputs)
for _ in range(NUM_LAYERS):
decoder_layer = TransformerDecoder(
intermediate_dim = INTERMEDIATE_DIM, num_heads= NUM_HEADS
)
x = decoder_layer(x)
x = Dropout(0.5)(x)
decoder_ouput = Dense(VOCAB_SIZE)(x)
transformer = Model(inputs=decoder_inputs, outputs=decoder_ouput, name="transformer")
loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
perplexity = Perplexity(from_logits=True, mask_token_id = 0)
transformer.compile(optimizer = "adam", loss="mse", metrics=[perplexity])
transformer.fit(train_ds, epochs=EPOCHS,batch_size=128 ,verbose=2 ,
validation_data=val_ds,callbacks=[model_checkpoint_callback])
AND THE ERROR
File "c:\users\hp\appdata\local\programs\python\python38\lib\site-
packages\keras\engine\compile_utils.py", line 265, in __call__
loss_value = loss_obj(y_t, y_p, sample_weight=sw)
File "c:\users\hp\appdata\local\programs\python\python38\lib\site-packages\keras\losses.py",
line 152, in __call__
losses = call_fn(y_true, y_pred)
File "c:\users\hp\appdata\local\programs\python\python38\lib\site-packages\keras\losses.py",
line 272, in call **
return ag_fn(y_true, y_pred, **self._fn_kwargs)
File "c:\users\hp\appdata\local\programs\python\python38\lib\site-packages\keras\losses.py",
line 2115, in _ragged_tensor_sparse_categorical_crossentropy
return _ragged_tensor_apply_loss(fn, y_true, y_pred, y_pred_extra_dim=True)
File "c:\users\hp\appdata\local\programs\python\python38\lib\site-packages\keras\losses.py",
line 1564, in _ragged_tensor_apply_loss
nested_splits_list = [rt.nested_row_splits for rt in (y_true, y_pred)]
File "c:\users\hp\appdata\local\programs\python\python38\lib\site-packages\keras\losses.py",
line 1564, in <listcomp>
nested_splits_list = [rt.nested_row_splits for rt in (y_true, y_pred)]
AttributeError: 'Tensor' object has no attribute 'nested_row_splits'