--train2
--n000002
--0001_0.jpg
--0001_02.jpg
--0001_03.jpg
--n000003
--n000004
ERROR
2020-01-08 09:29:18.958101: W tensorflow/core/framework/op_kernel.cc:1502] OP_REQUIRES failed at strided_slice_op.cc:107 : Invalid argument: slice index -1 of dimension 0 out of bounds.
2020-01-08 09:29:18.958102: W tensorflow/core/framework/op_kernel.cc:1502] OP_REQUIRES failed at strided_slice_op.cc:107 : Invalid argument: slice index -1 of dimension 0 out of bounds.
2020-01-08 09:29:18.958102: W tensorflow/core/framework/op_kernel.cc:1502] OP_REQUIRES failed at strided_slice_op.cc:107 : Invalid argument: slice index -1 of dimension 0 out of bounds.
2020-01-08 09:29:18.958109: W tensorflow/core/framework/op_kernel.cc:1502] OP_REQUIRES failed at strided_slice_op.cc:107 : Invalid argument: slice index -1 of dimension 0 out of bounds.
2020-01-08 09:29:18.958112: W tensorflow/core/framework/op_kernel.cc:1502] OP_REQUIRES failed at strided_slice_op.cc:107 : Invalid argument: slice index -1 of dimension 0 out of bounds.
2020-01-08 09:29:18.958114: W tensorflow/core/framework/op_kernel.cc:1502] OP_REQUIRES failed at strided_slice_op.cc:107 : Invalid argument: slice index -1 of dimension 0 out of bounds.
2020-01-08 09:29:18.958146: W tensorflow/core/framework/op_kernel.cc:1502] OP_REQUIRES failed at strided_slice_op.cc:107 : Invalid argument: slice index -1 of dimension 0 out of bounds.
2020-01-08 09:29:18.959258: W tensorflow/core/framework/op_kernel.cc:1502] OP_REQUIRES failed at strided_slice_op.cc:107 : Invalid argument: slice index -1 of dimension 0 out of bounds.
2020-01-08 09:29:18.985959: W tensorflow/core/framework/op_kernel.cc:1502] OP_REQUIRES failed at iterator_ops.cc:1037 : Invalid argument: slice index -1 of dimension 0 out of bounds.
[[{{node strided_slice}}]]
multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
File "C:\Users\Kmu\AppData\Local\Programs\Python\Python36\lib\multiprocessing\pool.py", line 119, in worker
result = (True, func(*args, **kwds))
File "C:\Users\Kmu\AppData\Local\Programs\Python\Python36\lib\multiprocessing\pool.py", line 44, in mapstar
return list(map(*args))
File "C:\Users\Kmu\PycharmProjects\deneme1\tfrecords-faster\tfrecsfaster\prepare_tfrecords2.py", line 73, in task
for shard_id, (image, label) in enumerate(dataset):
File "C:\Users\Kmu\PycharmProjects\deneme1\venv\lib\site-packages\tensorflow\python\data\ops\iterator_ops.py", line 586, in __next__
return self.next()
File "C:\Users\Kmu\PycharmProjects\deneme1\venv\lib\site-packages\tensorflow\python\data\ops\iterator_ops.py", line 623, in next
return self._next_internal()
File "C:\Users\Kmu\PycharmProjects\deneme1\venv\lib\site-packages\tensorflow\python\data\ops\iterator_ops.py", line 615, in _next_internal
output_shapes=self._flat_output_shapes)
File "C:\Users\Kmu\PycharmProjects\deneme1\venv\lib\site-packages\tensorflow\python\ops\gen_dataset_ops.py", line 2150, in iterator_get_next_sync
_six.raise_from(_core._status_to_exception(e.code, message), None)
File "<string>", line 3, in raise_from
tensorflow.python.framework.errors_impl.InvalidArgumentError: slice index -1 of dimension 0 out of bounds.
[[{{node strided_slice}}]] [Op:IteratorGetNextSync]
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:/Users/Kmu/PycharmProjects/deneme1/tfrecords-faster/tfrecsfaster/prepare_tfrecords2.py", line 92, in <module>
pool.map(task, inputs)
File "C:\Users\Kmu\AppData\Local\Programs\Python\Python36\lib\multiprocessing\pool.py", line 266, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File "C:\Users\Kmu\AppData\Local\Programs\Python\Python36\lib\multiprocessing\pool.py", line 644, in get
raise self._value
tensorflow.python.framework.errors_impl.InvalidArgumentError: slice index -1 of dimension 0 out of bounds.
[[{{node strided_slice}}]] [Op:IteratorGetNextSync]
Process finished with exit code 1
#!/usr/bin/env python3
import os
WORKERS = os.cpu_count()
print("Using %d workers." % WORKERS)
dataset_path = 'C:/Users/Kmu/PycharmProjects/train2'
tfrecs_path = 'C:/Users/Kmu/PycharmProjects/dosyalar/tfrecords'
shard_size = 12000
n_samples = None
def tensorflow():
import tensorflow as tf
tf.enable_eager_execution()
return tf
def init_task():
print('[init_task] Started...')
tf = tensorflow()
image_paths = tf.io.gfile.glob(dataset_path + '/*/*.jpg')
batch_size = len(image_paths) // WORKERS
if len(image_paths) % WORKERS != 0:
batch_size += 1
image_paths_batched = [
image_paths[index * batch_size: (index + 1) * batch_size]
for index in range(WORKERS)
]
global n_samples
n_samples = len(image_paths)
print('[init_task] Exiting...')
return image_paths_batched
def task(inputs):
worker_id, filenames = inputs
def print_info(text):
print('[Worker %d] %s' % (worker_id, text))
print_info('Started...')
tf = tensorflow()
def wrap_bytes(list_of_bytestrings):
return tf.train.Feature(bytes_list=tf.train.BytesList(value=list_of_bytestrings))
def get_image_and_class(image_path):
class_label = tf.strings.split(tf.expand_dims(image_path, axis=-1), sep='/').values[-2]
img = tf.io.read_file(image_path)
return img, class_label
def to_tfrecord(image, class_name):
feature = {
'image': wrap_bytes([image]),
'class': wrap_bytes([class_name])
}
return tf.train.Example(features=tf.train.Features(feature=feature))
filenames = tf.convert_to_tensor(filenames)
dataset = tf.data.Dataset.from_tensor_slices(filenames)
dataset = dataset.shuffle(n_samples // WORKERS)
dataset = dataset.map(get_image_and_class, tf.data.experimental.AUTOTUNE)
dataset = dataset.batch(shard_size)
for shard_id, (image, label) in enumerate(dataset):
print_info('Working on shard %d' % shard_id)
image, label = image.numpy(), label.numpy()
current_shard_limit = image.shape[0]
filename = tfrecs_path + '/worker{}-{:06d}-l{}.tfrec'.format(worker_id, shard_id, current_shard_limit)
with tf.io.TFRecordWriter(filename) as out_file:
for i in range(current_shard_limit):
record = to_tfrecord(image[i], label[i])
out_file.write(record.SerializeToString())
print_info('Wrote file {} containing {} records'.format(filename, current_shard_limit))
image_paths_batched = init_task()
inputs = [(index, batch) for index, batch in enumerate(image_paths_batched)]
import multiprocessing
if __name__ == '__main__':
with multiprocessing.Pool(WORKERS) as pool:
pool.map(task, inputs)
print('All workers exited')