Trying to load the data with a pytorch Dataloader using several processes (num_workers) makes pytables crash.
dataset = DL1DataReader(files,
selected_telescope_type='LST_LSTCam', selected_telescope_ids=[1],
image_channels=['charge', 'peakpos'],
event_info=['mc_energy', 'core_x', 'core_y', 'alt', 'az', 'shower_primary_id'],
array_info=['x', 'y', 'z'],
mapping_settings={'camera_types': ['LSTCam'],
'mapping_method': {'LSTCam': 'axial_addressing'}},
transforms=[ConvertShowerPrimaryIDToClassLabel(),
ImpactToKm(),
DataForGammaLearn()]
)
dl = DataLoader(dataset, batch_size=256, shuffle=True, drop_last=True, num_workers=4)
for i, batch in enumerate(dl):
print('batch {}'.format(i))
print(type(batch['image']))
print(batch['label'].shape)
---------------------------------------------------------------------------
HDF5ExtError Traceback (most recent call last)
<ipython-input-6-3293362984ef> in <module>()
----> 1 for i, batch in enumerate(dl):
2 print('batch {}'.format(i))
3 print(type(batch['image']))
4 print(batch['label'].shape)
~/anaconda3/envs/torch10/lib/python3.6/site-packages/torch/utils/data/dataloader.py in __next__(self)
635 self.reorder_dict[idx] = batch
636 continue
--> 637 return self._process_next_batch(batch)
638
639 next = __next__ # Python 2 compatibility
~/anaconda3/envs/torch10/lib/python3.6/site-packages/torch/utils/data/dataloader.py in _process_next_batch(self, batch)
656 self._put_indices()
657 if isinstance(batch, ExceptionWrapper):
--> 658 raise batch.exc_type(batch.exc_msg)
659 return batch
660
HDF5ExtError: Traceback (most recent call last):
File "/home/mikael/anaconda3/envs/torch10/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 138, in _worker_loop
samples = collate_fn([dataset[i] for i in batch_indices])
File "/home/mikael/anaconda3/envs/torch10/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 138, in <listcomp>
samples = collate_fn([dataset[i] for i in batch_indices])
File "/home/mikael/dl1-data-handler/dl1_data_handler/reader.py", line 395, in __getitem__
image = self._get_image(filename, self.tel_type, image_index)
File "/home/mikael/dl1-data-handler/dl1_data_handler/reader.py", line 341, in _get_image
in f.root.Telescope_Type_Information.where(query)][0]
File "/home/mikael/dl1-data-handler/dl1_data_handler/reader.py", line 340, in <listcomp>
length = [x['num_pixels'] for x
File "tables/tableextension.pyx", line 903, in tables.tableextension.Row.__next__
File "tables/tableextension.pyx", line 1084, in tables.tableextension.Row.__next__inkernel
File "tables/tableextension.pyx", line 577, in tables.tableextension.Table._read_records
tables.exceptions.HDF5ExtError: HDF5 error back trace
File "H5Dio.c", line 216, in H5Dread
can't read data
File "H5Dio.c", line 587, in H5D__read
can't read data
File "H5Dchunk.c", line 2276, in H5D__chunk_read
error looking up chunk address
File "H5Dchunk.c", line 3022, in H5D__chunk_lookup
can't query chunk address
File "H5Dbtree.c", line 1047, in H5D__btree_idx_get_addr
can't get chunk info
File "H5B.c", line 341, in H5B_find
unable to load B-tree node
File "H5AC.c", line 1763, in H5AC_protect
H5C_protect() failed
File "H5C.c", line 2565, in H5C_protect
can't load entry
File "H5C.c", line 6890, in H5C_load_entry
Can't deserialize image
File "H5Bcache.c", line 181, in H5B__cache_deserialize
wrong B-tree signature
End of HDF5 error back trace
Problems reading records.
The following minimal example also generates errors:
def read_dataset(num, dataset):
print(dataset)
for k in range(1000):
j = int(np.random.rand()*len(dataset))
data = dataset[j]
print('worker {} reads data, iteration {}'.format(num, k))
return
workers = [mp.Process(target=read_dataset, args=(i, dataset,)) for i in range(4)]
for w in workers:
w.start()
for w in workers:
w.join()
worker 3 reads data, iteration 64
worker 0 reads data, iteration 79
worker 1 reads data, iteration 72
Process Process-9:
worker 2 reads data, iteration 66
Process Process-10:
Traceback (most recent call last):
worker 3 reads data, iteration 65
File "/home/mikael/anaconda3/envs/torch10/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
self.run()
Traceback (most recent call last):
worker 2 reads data, iteration 67
File "/home/mikael/anaconda3/envs/torch10/lib/python3.6/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
worker 3 reads data, iteration 66
File "<ipython-input-26-dae7f6b79d79>", line 6, in read_dataset
data = dataset[j]
File "/home/mikael/dl1-data-handler/dl1_data_handler/reader.py", line 393, in __getitem__
nrow = f.root._f_get_child(self.tel_type)[image_index]['event_index']
File "/home/mikael/anaconda3/envs/torch10/lib/python3.6/site-packages/tables/table.py", line 2079, in __getitem__
return self.read(start, stop, step)[0]
File "/home/mikael/anaconda3/envs/torch10/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/home/mikael/anaconda3/envs/torch10/lib/python3.6/site-packages/tables/table.py", line 1934, in read
arr = self._read(start, stop, step, field, out)
worker 3 reads data, iteration 67
File "/home/mikael/anaconda3/envs/torch10/lib/python3.6/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "/home/mikael/anaconda3/envs/torch10/lib/python3.6/site-packages/tables/table.py", line 1848, in _read
self._read_records(start, stop - start, result)
File "<ipython-input-26-dae7f6b79d79>", line 6, in read_dataset
data = dataset[j]
File "tables/tableextension.pyx", line 577, in tables.tableextension.Table._read_records
File "/home/mikael/dl1-data-handler/dl1_data_handler/reader.py", line 393, in __getitem__
nrow = f.root._f_get_child(self.tel_type)[image_index]['event_index']
worker 3 reads data, iteration 68
tables.exceptions.HDF5ExtError: HDF5 error back trace
File "H5Dio.c", line 216, in H5Dread
can't read data
File "H5Dio.c", line 587, in H5D__read
can't read data
File "H5Dchunk.c", line 2304, in H5D__chunk_read
unable to read raw data chunk
File "H5Dchunk.c", line 3659, in H5D__chunk_lock
data pipeline read failed
File "H5Z.c", line 1279, in H5Z_pipeline
filter returned failure during read
End of HDF5 error back trace
Problems reading records.
File "/home/mikael/anaconda3/envs/torch10/lib/python3.6/site-packages/tables/table.py", line 2079, in __getitem__
return self.read(start, stop, step)[0]
File "/home/mikael/anaconda3/envs/torch10/lib/python3.6/site-packages/tables/table.py", line 1934, in read
arr = self._read(start, stop, step, field, out)
File "/home/mikael/anaconda3/envs/torch10/lib/python3.6/site-packages/tables/table.py", line 1848, in _read
self._read_records(start, stop - start, result)
File "tables/tableextension.pyx", line 577, in tables.tableextension.Table._read_records
tables.exceptions.HDF5ExtError: HDF5 error back trace
File "H5Dio.c", line 216, in H5Dread
can't read data
File "H5Dio.c", line 587, in H5D__read
can't read data
File "H5Dchunk.c", line 2304, in H5D__chunk_read
unable to read raw data chunk
File "H5Dchunk.c", line 3659, in H5D__chunk_lock
data pipeline read failed
File "H5Z.c", line 1279, in H5Z_pipeline
filter returned failure during read
End of HDF5 error back trace
Problems reading records.
While the following example using Threading works:
def worker(num, dataset):
print(dataset)
for k in range(1000):
j = int(np.random.rand()*len(dataset))
data = dataset[j]
print('worker {} reads data, iteration {}'.format(num, k))
return
threads = []
for i in range(4):
t = threading.Thread(target=worker, args=(i, dataset,))
threads.append(t)
t.start()
worker 0 reads data, iteration 999
worker 1 reads data, iteration 991
worker 3 reads data, iteration 987
worker 2 reads data, iteration 987
worker 1 reads data, iteration 992worker 3 reads data, iteration 988
worker 2 reads data, iteration 988
worker 3 reads data, iteration 989
worker 1 reads data, iteration 993worker 2 reads data, iteration 989
worker 1 reads data, iteration 994
worker 3 reads data, iteration 990
worker 1 reads data, iteration 995
worker 3 reads data, iteration 991
worker 2 reads data, iteration 990
worker 3 reads data, iteration 992
worker 1 reads data, iteration 996
worker 3 reads data, iteration 993
worker 1 reads data, iteration 997
worker 3 reads data, iteration 994
worker 2 reads data, iteration 991
worker 1 reads data, iteration 998
worker 3 reads data, iteration 995
worker 1 reads data, iteration 999worker 2 reads data, iteration 992
worker 3 reads data, iteration 996
worker 2 reads data, iteration 993
worker 3 reads data, iteration 997
worker 2 reads data, iteration 994
worker 3 reads data, iteration 998
worker 3 reads data, iteration 999
worker 2 reads data, iteration 995
worker 2 reads data, iteration 996
worker 2 reads data, iteration 997
worker 2 reads data, iteration 998
worker 2 reads data, iteration 999
Threading in python doesn't seem to allow true concurrency though, because of the GIL ( Global Interpreter Lock).
Loading batches in a single process may slow down the training.