I have a super resolution dataset which have lr and hr folder containing png images.
I follow the instruction on the site and based on div2k srcipt to write a load for my dataset.
My training cannot load the dataset with the tfds, and it tried to download some other dataset, i.e. I comment the download action.
error : Downloading and preparing dataset Unknown size (download: Unknown size, generated: Unknown size, total: Unknown size) to /root/tensorflow_datasets/grass/1.0.0…
TypeError: _generate_examples() got an unexpected keyword argument ‘lr_path’
class Grass(tfds.core.GeneratorBasedBuilder):
"""DatasetBuilder for face_grass dataset."""
VERSION = tfds.core.Version('1.0.0')
RELEASE_NOTES = {
'1.0.0': 'Initial release.',
}
def _info(self) -> tfds.core.DatasetInfo:
"""Returns the dataset metadata."""
# TODO(face_grass): Specifies the tfds.core.DatasetInfo object
return tfds.core.DatasetInfo(
builder=self,
description=_DESCRIPTION,
features=tfds.features.FeaturesDict({
"lr": tfds.features.Image(),
"hr": tfds.features.Image(),
}),
supervised_keys=("lr", "hr"),
# If there's a common (input, target) tuple from the
# features, specify them here. They'll be used if
# `as_supervised=True` in `builder.as_dataset`.
# supervised_keys=('image', 'label'), # Set to `None` to disable
homepage='https://dataset-homepage/',
citation=_CITATION,
)
def _split_generators(self, dl_manager: tfds.download.DownloadManager):
"""Returns SplitGenerators."""
# TODO(face_grass): Downloads the data and defines the splits
#path = dl_manager.download_and_extract('https://todo-data-url')
# TODO(face_grass): Returns the Dict[split names, Iterator[Key, Example]]
return [
tfds.core.SplitGenerator(
name=tfds.Split.TRAIN,
gen_kwargs={
"lr_path":
"../data1/project/grass_0808/train/LR",
"hr_path":
"../data1/project/grass_0808/train/HR",
}),
tfds.core.SplitGenerator(
name=tfds.Split.VALIDATION,
gen_kwargs={
"lr_path":
"../data1/project/grass_0808/valid/LR",
"hr_path":
"../data1/project/grass_0808/valid/HR",
}),
]
def _generate_examples(self, path):
"""Yields examples."""
# TODO(face_grass): Yields (key, example) tuples from the dataset
for root, _, files in tf.io.gfile.walk(lr_path):
for file_path in files:
# Select only png files.
if file_path.endswith(".png"):
yield file_path, {
"lr": os.path.join(root, file_path),
"hr": os.path.join(hr_path, file_path)
}
Thanks.