After successfully running the processing, assuming your response to the other thread is that it's alright, I got into the following problem with running the training:
!CUDA_VISIBLE_DEVICES=0 pathflowai-train_model train_model --prediction --patch_size 512 -pr 224 --save_location outcomes_model.pkl -a resnet34 --input_dir /project/PFAI_inputs -nt 1 -t 10000 -lr 1e-4 -ne 10 -ss 0.5 -ssv 0.3 -tt 0.1 -bt 0.01 -imb -pi patch_information.db -bs 32 -ca
Error: mkl-service + Intel(R) MKL: MKL_THREADING_LAYER=INTEL is incompatible with libgomp.so.1 library.
Try to import numpy first or set the threading layer accordingly. Set MKL_SERVICE_FORCE_INTEL to force it.
(saturn) jovyan@jupyter-assafmagen-2dpathflowai:~/project$ CUDA_VISIBLE_DEVICES=0 pathflowai-train_model train_model --prediction --patch_size 512 -pr 224 --save_location outcomes_model.pkl -a resnet34 --input_dir /project/PFAI_inputs/ -nt 1 -t 10000 -lr 1e-4 -ne 10 -ss 0.5 -ssv 0.3 -tt 0.1 -bt 0.01 -imb -pi patch_information.db -bs 32 -ca
nonechucks may not work properly with this version of PyTorch (1.5.0). It has only been tested on PyTorch versions 1.0, 1.1, and 1.2
/srv/conda/envs/saturn/lib/python3.6/site-packages/pathflowai/utils.py:605: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead
See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
()
Traceback (most recent call last):
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/fsspec/mapping.py", line 76, in __getitem__
result = self.fs.cat(k)
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/fsspec/spec.py", line 587, in cat
return self.open(path, "rb").read()
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/fsspec/spec.py", line 774, in open
**kwargs
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/fsspec/implementations/local.py", line 108, in _open
return LocalFileOpener(path, mode, fs=self, **kwargs)
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/fsspec/implementations/local.py", line 175, in __init__
self._open()
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/fsspec/implementations/local.py", line 180, in _open
self.f = open(self.path, mode=self.mode)
FileNotFoundError: [Errno 2] No such file or directory: '/project/PFAI_inputs/Li63N2DCLAMP.zarr/.zarray'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/zarr/core.py", line 150, in _load_metadata_nosync
meta_bytes = self._store[mkey]
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/fsspec/mapping.py", line 80, in __getitem__
raise KeyError(key)
KeyError: '.zarray'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/srv/conda/envs/saturn/bin/pathflowai-train_model", line 8, in <module>
sys.exit(train())
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/click/core.py", line 722, in __call__
return self.main(*args, **kwargs)
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/click/core.py", line 697, in main
rv = self.invoke(ctx)
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/click/core.py", line 1066, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/click/core.py", line 895, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/click/core.py", line 535, in invoke
return callback(*args, **kwargs)
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/pathflowai/model_training.py", line 309, in train_model
train_model_(training_opts)
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/pathflowai/model_training.py", line 37, in train_model_
norm_dict = get_normalizer(training_opts['normalization_file'], dataset_opts)
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/pathflowai/datasets.py", line 168, in get_normalizer
dataset = DynamicImageDataset(**dataset_opts)#nc.SafeDataset(DynamicImageDataset(**dataset_opts))
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/pathflowai/datasets.py", line 320, in __init__
self.slides = {slide:da.from_zarr(join(input_dir,'{}.zarr'.format(slide))) for slide in IDs}
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/pathflowai/datasets.py", line 320, in <dictcomp>
self.slides = {slide:da.from_zarr(join(input_dir,'{}.zarr'.format(slide))) for slide in IDs}
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/dask/array/core.py", line 2842, in from_zarr
z = zarr.Array(mapper, read_only=True, path=component, **kwargs)
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/zarr/core.py", line 124, in __init__
self._load_metadata()
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/zarr/core.py", line 141, in _load_metadata
self._load_metadata_nosync()
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/zarr/core.py", line 152, in _load_metadata_nosync
err_array_not_found(self._path)
File "/srv/conda/envs/saturn/lib/python3.6/site-packages/zarr/errors.py", line 25, in err_array_not_found
raise ValueError('array not found at path %r' % path)
ValueError: array not found at path ''