Hi,
In the latest wholeslidedata version, calling create_batch_iterator with a copy_path specified in the user_config, and a number_of_batches > 0, causes an error. It seems to be some sort of multiprocessing issue (even though it also occurs when cpus==1), in which one process tries to open an image while another process is still copying the file. I don't know why this is happening, especially with 1 cpu specified, but I think that moving the dataset creation call in create_batch_iterator to before the commander, producer, and buffer_factory are created should fix it. If there are reasons as to why the dataset creation should be afterwards in the case of number_of_batches>0, please let me know.
Example error stacktrace:
Process ProducerForkProcess-4:
Process ProducerForkProcess-11:
Process ProducerForkProcess-5:
Traceback (most recent call last):
Traceback (most recent call last):
Traceback (most recent call last):
File "/usr/local/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
self.run()
Process ProducerForkProcess-6:
File "/usr/local/lib/python3.9/site-packages/concurrentbuffer/producer.py", line 71, in run
self._producer.build()
Process ProducerForkProcess-10:
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/buffer/batchproducer.py", line 22, in build
builds = build_config(self._config[self._mode])
Process CommanderForkProcess-3:
Traceback (most recent call last):
Process ProducerForkProcess-7:
File "/usr/local/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
self.run()
File "/usr/local/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
self.run()
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 124, in build_config
return _ObjectFactory(deepcopy(config)).build_config()
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 26, in build_config
return self._build(self._configuration)
File "/usr/local/lib/python3.9/site-packages/concurrentbuffer/producer.py", line 71, in run
self._producer.build()
File "/usr/local/lib/python3.9/site-packages/concurrentbuffer/producer.py", line 71, in run
self._producer.build()
File "/usr/local/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
self.run()
Traceback (most recent call last):
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/buffer/batchproducer.py", line 22, in build
builds = build_config(self._config[self._mode])
File "/usr/local/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
self.run()
File "/usr/local/lib/python3.9/functools.py", line 938, in _method
return method.get(obj, cls)(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/buffer/batchproducer.py", line 22, in build
builds = build_config(self._config[self._mode])
File "/usr/local/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
self.run()
File "/usr/local/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
self.run()
File "/usr/local/lib/python3.9/site-packages/concurrentbuffer/producer.py", line 71, in run
self._producer.build()
File "/usr/local/lib/python3.9/site-packages/concurrentbuffer/commander.py", line 66, in run
self._commander.build()
File "/usr/local/lib/python3.9/site-packages/concurrentbuffer/producer.py", line 71, in run
self._producer.build()
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 38, in _build_dict
config[key] = self._build_object(value)
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 124, in build_config
return _ObjectFactory(deepcopy(config)).build_config()
File "/usr/local/lib/python3.9/site-packages/concurrentbuffer/producer.py", line 71, in run
self._producer.build()
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/buffer/batchproducer.py", line 22, in build
builds = build_config(self._config[self._mode])
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 124, in build_config
return _ObjectFactory(deepcopy(config)).build_config()
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/buffer/batchcommander.py", line 26, in build
builds = build_config(self._config[self._mode])
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 124, in build_config
return _ObjectFactory(deepcopy(config)).build_config()
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 124, in build_config
return _ObjectFactory(deepcopy(config)).build_config()
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 26, in build_config
return self._build(self._configuration)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/buffer/batchproducer.py", line 22, in build
builds = build_config(self._config[self._mode])
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 67, in _build_object
return attribute(*args, **kwargs)
File "/usr/local/lib/python3.9/functools.py", line 938, in _method
return method.get(obj, cls)(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 124, in build_config
return _ObjectFactory(deepcopy(config)).build_config()
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/dataset.py", line 42, in init
self._data = dict(sorted(self._open(self._associations, labels=labels).items()))
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 38, in _build_dict
config[key] = self._build_object(value)
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 26, in build_config
return self._build(self._configuration)
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 26, in build_config
return self._build(self._configuration)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/dataset.py", line 94, in _open
image, spacing = self._open_image(wsi_file)
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 67, in _build_object
return attribute(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 26, in build_config
return self._build(self._configuration)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/buffer/batchproducer.py", line 22, in build
builds = build_config(self._config[self._mode])
File "/usr/local/lib/python3.9/functools.py", line 938, in _method
return method.get(obj, cls)(*args, **kwargs)
File "/usr/local/lib/python3.9/functools.py", line 938, in _method
return method.get(obj, cls)(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 124, in build_config
return _ObjectFactory(deepcopy(config)).build_config()
File "/usr/local/lib/python3.9/functools.py", line 938, in _method
return method.get(obj, cls)(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/dataset.py", line 116, in _open_image
wsi = wsi_file.open()
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 38, in _build_dict
config[key] = self._build_object(value)
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 26, in build_config
return self._build(self._configuration)
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 38, in _build_dict
config[key] = self._build_object(value)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/files.py", line 34, in open
return WholeSlideImage(self.path, self._image_backend)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/dataset.py", line 42, in init
self._data = dict(sorted(self._open(self._associations, labels=labels).items()))
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 38, in _build_dict
config[key] = self._build_object(value)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/image/wholeslideimage.py", line 35, in init
self._backend = get_backend(backend)(path=self._path)
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 67, in _build_object
return attribute(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 67, in _build_object
return attribute(*args, **kwargs)
File "/usr/local/lib/python3.9/functools.py", line 938, in _method
return method.get(obj, cls)(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/interoperability/asap/backend.py", line 15, in init
raise ValueError(f"cant open image {path}")
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 26, in build_config
return self._build(self._configuration)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/dataset.py", line 42, in init
self._data = dict(sorted(self._open(self._associations, labels=labels).items()))
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/dataset.py", line 94, in _open
image, spacing = self._open_image(wsi_file)
ValueError: cant open image /home/user/tmp/images/0014_R.tif
File "/usr/local/lib/python3.9/functools.py", line 938, in _method
return method.get(obj, cls)(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 38, in _build_dict
config[key] = self._build_object(value)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/dataset.py", line 116, in _open_image
wsi = wsi_file.open()
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/dataset.py", line 94, in _open
image, spacing = self._open_image(wsi_file)
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 67, in _build_object
return attribute(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 67, in _build_object
return attribute(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/dataset.py", line 42, in init
self._data = dict(sorted(self._open(self._associations, labels=labels).items()))
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/dataset.py", line 42, in init
self._data = dict(sorted(self._open(self._associations, labels=labels).items()))
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/dataset.py", line 42, in init
self._data = dict(sorted(self._open(self._associations, labels=labels).items()))
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/dataset.py", line 94, in _open
image, spacing = self._open_image(wsi_file)
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 38, in _build_dict
config[key] = self._build_object(value)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/dataset.py", line 94, in _open
image, spacing = self._open_image(wsi_file)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/files.py", line 34, in open
return WholeSlideImage(self.path, self._image_backend)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/dataset.py", line 94, in _open
image, spacing = self._open_image(wsi_file)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/dataset.py", line 116, in _open_image
wsi = wsi_file.open()
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/dataset.py", line 116, in _open_image
wsi = wsi_file.open()
File "/usr/local/lib/python3.9/site-packages/dicfg/factory.py", line 67, in _build_object
return attribute(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/dataset.py", line 116, in _open_image
wsi = wsi_file.open()
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/image/wholeslideimage.py", line 35, in init
self._backend = get_backend(backend)(path=self._path)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/files.py", line 34, in open
return WholeSlideImage(self.path, self._image_backend)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/dataset.py", line 116, in _open_image
wsi = wsi_file.open()
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/files.py", line 34, in open
return WholeSlideImage(self.path, self._image_backend)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/image/wholeslideimage.py", line 35, in init
self._backend = get_backend(backend)(path=self._path)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/interoperability/asap/backend.py", line 15, in init
raise ValueError(f"cant open image {path}")
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/files.py", line 34, in open
return WholeSlideImage(self.path, self._image_backend)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/dataset.py", line 42, in init
self._data = dict(sorted(self._open(self._associations, labels=labels).items()))
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/image/wholeslideimage.py", line 35, in init
self._backend = get_backend(backend)(path=self._path)
ValueError: cant open image /home/user/tmp/images/0014_R.tif
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/files.py", line 34, in open
return WholeSlideImage(self.path, self._image_backend)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/interoperability/asap/backend.py", line 15, in init
raise ValueError(f"cant open image {path}")
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/image/wholeslideimage.py", line 35, in init
self._backend = get_backend(backend)(path=self._path)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/interoperability/asap/backend.py", line 15, in init
raise ValueError(f"cant open image {path}")
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/dataset.py", line 94, in _open
image, spacing = self._open_image(wsi_file)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/image/wholeslideimage.py", line 35, in init
self._backend = get_backend(backend)(path=self._path)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/interoperability/asap/backend.py", line 15, in init
raise ValueError(f"cant open image {path}")
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/interoperability/asap/backend.py", line 15, in init
raise ValueError(f"cant open image {path}")
ValueError: cant open image /home/user/tmp/images/0014_R.tif
ValueError: cant open image /home/user/tmp/images/0014_R.tif
ValueError: cant open image /home/user/tmp/images/0014_R.tif
ValueError: cant open image /home/user/tmp/images/0014_R.tif
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/dataset.py", line 116, in _open_image
wsi = wsi_file.open()
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/data/files.py", line 34, in open
return WholeSlideImage(self.path, self._image_backend)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/image/wholeslideimage.py", line 35, in init
self._backend = get_backend(backend)(path=self._path)
File "/usr/local/lib/python3.9/site-packages/wholeslidedata/interoperability/asap/backend.py", line 15, in init
raise ValueError(f"cant open image {path}")
ValueError: cant open image /home/user/tmp/images/0014_R.tif