minimaxir / automl-gs Goto Github PK

ValueError: Parent directory of model_weights.hdf5 doesn't exist, can't save.

Solving a binary_classification problem, maximizing accuracy using tensorflow.

Modeling with field specifications:
Subject: categorical
Finished: categorical
TrainingDay: categorical
Condition: categorical
CondPrev: categorical
TaskNumber: categorical
TaskId: categorical
TrialNumber: numeric
PresentationStimulus: numeric
StimTime: numeric
RespToTime: numeric
RT: numeric
SubjResp: categorical
OutcomeInt: categorical
TaskOutcomeInt: categorical
StimDim1: categorical
StimDim2: categorical
StimDim3: categorical
StimDim4: categorical
IntendedRule: categorical
Background: categorical
StimDimWord1: categorical
StimDimWord2: categorical
StimDimWord3: categorical
StimDimWord4: categorical
ExpResp: categorical
DistinctDays: categorical
out: categorical
StimType: categorical
0% 0/100 [00:00<?, ?trial/s]
0% 0/20 [00:00<?, ?epoch/s]
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-16-ca69e1157d4e> in <module>()
      2                    target_field='correctResp',
      3                    model_name='tpu',
----> 4                    tpu_address = tpu_address)

/usr/local/lib/python3.6/dist-packages/automl_gs/automl_gs.py in automl_grid_search(csv_path, target_field, target_metric, framework, model_name, context, num_trials, split, num_epochs, col_types, gpu, tpu_address)
     92                     header=(best_result is None))
     93 
---> 94         train_results = results.tail(1).to_dict('records')[0]
     95 
     96         # If the target metric improves, save the new hps/files,

IndexError: list index out of range

Apr 3, 2019, 5:31:44 PM | WARNING | ValueError: Input contains NaN, infinity or a value too large for dtype('float32').
-- | -- | --
Apr 3, 2019, 5:31:44 PM | WARNING | raise ValueError(msg_err.format(type_err, X.dtype))
Apr 3, 2019, 5:31:44 PM | WARNING | File "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py", line 56, in _assert_all_finite
Apr 3, 2019, 5:31:44 PM | WARNING | allow_nan=force_all_finite == 'allow-nan')
Apr 3, 2019, 5:31:44 PM | WARNING | File "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py", line 573, in check_array
Apr 3, 2019, 5:31:44 PM | WARNING | y_pred = check_array(y_pred, ensure_2d=False)
Apr 3, 2019, 5:31:44 PM | WARNING | File "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py", line 1763, in log_loss
Apr 3, 2019, 5:31:44 PM | WARNING | logloss = log_loss(y_true, y_pred)
Apr 3, 2019, 5:31:44 PM | WARNING | File "/content/tpu_train/pipeline.py", line 1126, in on_epoch_end
Apr 3, 2019, 5:31:44 PM | WARNING | callback.on_epoch_end(epoch, logs)
Apr 3, 2019, 5:31:44 PM | WARNING | File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/callbacks.py", line 251, in on_epoch_end
Apr 3, 2019, 5:31:44 PM | WARNING | callbacks.on_epoch_end(epoch, epoch_logs)
Apr 3, 2019, 5:31:44 PM | WARNING | File "/usr/local/lib/python3.6/dist-packages/tensorflow/contrib/tpu/python/tpu/keras_support.py", line 1734, in _pipeline_fit_loop
Apr 3, 2019, 5:31:44 PM | WARNING | validation_steps=validation_steps)
Apr 3, 2019, 5:31:44 PM | WARNING | File "/usr/local/lib/python3.6/dist-packages/tensorflow/contrib/tpu/python/tpu/keras_support.py", line 1633, in _pipeline_fit
Apr 3, 2019, 5:31:44 PM | WARNING | steps_per_epoch, validation_steps, **kwargs)
Apr 3, 2019, 5:31:44 PM | WARNING | File "/usr/local/lib/python3.6/dist-packages/tensorflow/contrib/tpu/python/tpu/keras_support.py", line 1532, in fit
Apr 3, 2019, 5:31:44 PM | WARNING | batch_size=64 * 8)
Apr 3, 2019, 5:31:44 PM | WARNING | File "/content/tpu_train/pipeline.py", line 1095, in model_train
Apr 3, 2019, 5:31:44 PM | WARNING | model_train(df, encoders, args, model)
Apr 3, 2019, 5:31:44 PM | WARNING | File "model.py", line 69, in <module>
Apr 3, 2019, 5:31:44 PM | WARNING | Traceback (most recent call last):

          from automl_gs import automl_grid_search
          automl_grid_search('Housing.csv','price')

Traceback (most recent call last):
  File "model.py", line 59, in <module>
    model_train(df, encoders, args, model)
  File "C:\Users\josep\automl_train\pipeline.py", line 835, in model_train
    batch_size=256)
  File "C:\Users\josep\venv\lib\site-packages\tensorflow\python\keras\engine\training.py", line 776, in fit
    shuffle=shuffle)
  File "C:\Users\josep\venv\lib\site-packages\tensorflow\python\keras\engine\training.py", line 2382, in _standardize_user_data
    exception_prefix='input')
  File "C:\Users\josep\venv\lib\site-packages\tensorflow\python\keras\engine\training_utils.py", line 362, in standardize_input_data
    ' but got array with shape ' + str(data_shape))
ValueError: Error when checking input: expected input_son to have shape (1,) but got array with shape (2,)

    input_son_size = len(encoders['son_encoder'].classes_)
    input_son = Input(
        shape=(input_son_size if input_son_size != 2 else 1,), name="input_son")

/Users/jberman/.local/share/virtualenvs/automl/lib/python3.7/importlib/_bootstrap.py:219: ImportWarning: can't resolve package from __spec__ or __package__, falling back on __name__ and __path__
  return f(*args, **kwds)
/Users/jberman/.local/share/virtualenvs/automl/lib/python3.7/site-packages/jinja2/utils.py:485: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working
  from collections import MutableMapping
/Users/jberman/.local/share/virtualenvs/automl/lib/python3.7/site-packages/jinja2/runtime.py:318: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working
  from collections import Mapping

  0%|          | 0/100 [00:00<?, ?trial/s]

  0%|          | 0/20 [00:00<?, ?epoch/s]�[ATraceback (most recent call last):
  File "model.py", line 2, in <module>
    import pandas as pd
ModuleNotFoundError: No module named 'pandas'


  0%|          | 0/20 [00:00<?, ?epoch/s]�[A/usr/local/Cellar/python/3.7.3/Frameworks/Python.framework/Versions/3.7/lib/python3.7/subprocess.py:858: ResourceWarning: subprocess 12646 is still running
  ResourceWarning, source=self)
ResourceWarning: Enable tracemalloc to get the object allocation traceback
Solving a classification problem, maximizing accuracy using tensorflow.

Modeling with field specifications:
message.received: numeric
message.sender: categorical
message.subject: text
message.size: numeric
message.recipients: categorical
response.sent: numeric
response.sender: categorical
response.subject: text
response.size: numeric
response.recipients: categorical
lag_readable: text
Traceback (most recent call last):
  File "/Users/jberman/.local/bin/automl_gs", line 10, in <module>
    sys.exit(cmd())
  File "/Users/jberman/.local/share/virtualenvs/automl/lib/python3.7/site-packages/automl_gs/automl_gs.py", line 175, in cmd
    tpu_address=args.tpu_address)
  File "/Users/jberman/.local/share/virtualenvs/automl/lib/python3.7/site-packages/automl_gs/automl_gs.py", line 87, in automl_grid_search
    "metadata", "results.csv"))
  File "/Users/jberman/.local/share/virtualenvs/automl/lib/python3.7/site-packages/pandas/io/parsers.py", line 702, in parser_f
    return _read(filepath_or_buffer, kwds)
  File "/Users/jberman/.local/share/virtualenvs/automl/lib/python3.7/site-packages/pandas/io/parsers.py", line 429, in _read
    parser = TextFileReader(filepath_or_buffer, **kwds)
  File "/Users/jberman/.local/share/virtualenvs/automl/lib/python3.7/site-packages/pandas/io/parsers.py", line 895, in __init__
    self._make_engine(self.engine)
  File "/Users/jberman/.local/share/virtualenvs/automl/lib/python3.7/site-packages/pandas/io/parsers.py", line 1122, in _make_engine
    self._engine = CParserWrapper(self.f, **self.options)
  File "/Users/jberman/.local/share/virtualenvs/automl/lib/python3.7/site-packages/pandas/io/parsers.py", line 1853, in __init__
    self._reader = parsers.TextReader(src, **kwds)
  File "pandas/_libs/parsers.pyx", line 387, in pandas._libs.parsers.TextReader.__cinit__
  File "pandas/_libs/parsers.pyx", line 705, in pandas._libs.parsers.TextReader._setup_parser_source
FileNotFoundError: [Errno 2] File b'automl_train/metadata/results.csv' does not exist: b'automl_train/metadata/results.csv'
sys:1: ResourceWarning: unclosed file <_io.TextIOWrapper name='automl_results.csv' mode='w' encoding='UTF-8'>
ResourceWarning: Enable tracemalloc to get the object allocation traceback

⊙  head automl_train/model.py                                                                                                jberman@USNYHJBERMANMB2 ●
import argparse
import pandas as pd
from pipeline import *

Traceback (most recent call last):
  File "model.py", line 3, in <module>
    from pipeline import *
  File "[MY_PATH]/automl_train/pipeline.py", line 1090
    1stflrsf_enc = df['1stFlrSF']
               ^
SyntaxError: invalid syntax

  0%|          | 0/20 [00:00<?, ?epoch/s]Traceback (most recent call last):
  File "[MY_PATH]/test_auto_ml/Test.py", line 8, in <module>
    do_the_thing("[MY_DATASET_PATH]/train.csv","SalePrice")
  File "[MY_PATH]/test_auto_ml/Test.py", line 5, in do_the_thing
    automl_grid_search(path,label)
  File "[MY_PYTHON_PATH]/site-packages/automl_gs/automl_gs.py", line 94, in automl_grid_search
    train_results = results.tail(1).to_dict('records')[0]
IndexError: list index out of range

(env) (base) C:\Users\josep\Jeenee\AutoML\automl_train>python model.py -d ..\automl-testbench\yacht-hydrodynamics\data.csv -m train
Traceback (most recent call last):
  File "model.py", line 46, in <module>
    model_train(df, encoders, args, model)
  File "C:\Users\josep\Jeenee\AutoML\automl_train\pipeline.py", line 347, in model_train
    X, y = process_data(df, encoders)
  File "C:\Users\josep\Jeenee\AutoML\automl_train\pipeline.py", line 296, in process_data
    df['Length-beam ratio'].values, encoders['length_beam_ratio_bins'], labels=False, include_lowest=True, duplicates='drop')
  File "C:\Users\josep\Jeenee\AutoML\venv\lib\site-packages\pandas\core\reshape\tile.py", line 235, in cut
    raise ValueError('bins must increase monotonically.')
ValueError: bins must increase monotonically.

    # Length-beam ratio
    length_beam_ratio_enc = df['Length-beam ratio']
    length_beam_ratio_bins = length_beam_ratio_enc.quantile(
        np.linspace(0, 1, 10+1))
    encoders['length_beam_ratio_bins'] = length_beam_ratio_bins
    
    # ....

    # Length-beam ratio
    length_beam_ratio_enc = pd.cut(
        df['Length-beam ratio'].values, encoders['length_beam_ratio_bins'], labels=False, include_lowest=True, duplicates='drop')

>>> encoders['length_beam_ratio_bins']
[2.73, 2.76, 3.15, 3.15, 3.1499999999999995, 3.15, 3.17, 3.32, 3.51, 3.51, 3.64]

ValueError: The least populated class in y has only 1 member, which is too few. The minimum number of groups for any class cannot be less than 2.

>$ automl_gs square.csv square
/usr/local/lib/python3.7/site-packages/automl_gs/utils_automl.py:270: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.
  metrics = yaml.load(f)
Solving a classification problem, maximizing accuracy using tensorflow.

Modeling with field specifications:
real: numeric
fake1: numeric
fake2: numeric
fake3: numeric
fake4: numeric
text: categorical
bool: categorical
/usr/local/lib/python3.7/site-packages/automl_gs/utils_automl.py:126: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.
  hps = yaml.load(f)
  0%|                                                                                                                                                                         | 0/100 [00:00<?, ?trial/s/usr/local/lib/python3.7/site-packages/automl_gs/utils_automl.py:199: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.
  metrics = yaml.load(f)[problem_type]
Traceback (most recent call last):
  File "model.py", line 47, in <module>
    model_train(df, encoders, args, model)
  File "../automodel/automl_train/pipeline.py", line 408, in model_train
    for train_indices, val_indices in split.split(np.zeros(y.shape[0]), y):
  File "/usr/local/lib/python3.7/site-packages/sklearn/model_selection/_split.py", line 1315, in split
    for train, test in self._iter_indices(X, y, groups):
  File "/usr/local/lib/python3.7/site-packages/sklearn/model_selection/_split.py", line 1695, in _iter_indices
    raise ValueError("The least populated class in y has only 1"
ValueError: The least populated class in y has only 1 member, which is too few. The minimum number of groups for any class cannot be less than 2.
                                                                                                                                                                                                        Traceback (most recent call last):                                                                                                                                              | 0/20 [00:00<?, ?epoch/s]
  File "/usr/local/bin/automl_gs", line 10, in <module>
    sys.exit(cmd())
  File "/usr/local/lib/python3.7/site-packages/automl_gs/automl_gs.py", line 175, in cmd
    tpu_address=args.tpu_address)
  File "/usr/local/lib/python3.7/site-packages/automl_gs/automl_gs.py", line 87, in automl_grid_search
    "metadata", "results.csv"))
  File "/usr/local/lib/python3.7/site-packages/pandas/io/parsers.py", line 702, in parser_f
    return _read(filepath_or_buffer, kwds)
  File "/usr/local/lib/python3.7/site-packages/pandas/io/parsers.py", line 429, in _read
    parser = TextFileReader(filepath_or_buffer, **kwds)
  File "/usr/local/lib/python3.7/site-packages/pandas/io/parsers.py", line 895, in __init__
    self._make_engine(self.engine)
  File "/usr/local/lib/python3.7/site-packages/pandas/io/parsers.py", line 1122, in _make_engine
    self._engine = CParserWrapper(self.f, **self.options)
  File "/usr/local/lib/python3.7/site-packages/pandas/io/parsers.py", line 1853, in __init__
    self._reader = parsers.TextReader(src, **kwds)
  File "pandas/_libs/parsers.pyx", line 387, in pandas._libs.parsers.TextReader.__cinit__
  File "pandas/_libs/parsers.pyx", line 705, in pandas._libs.parsers.TextReader._setup_parser_source
FileNotFoundError: [Errno 2] File b'automl_train/metadata/results.csv' does not exist: b'automl_train/metadata/results.csv'

/anaconda3/envs/automl-gs/lib/python3.6/site-packages/automl_gs/utils_automl.py:270:
YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default 
Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.
  metrics = yaml.load(f)

---------------------------------------------------------------------------

FileNotFoundError                         Traceback (most recent call last)

<ipython-input-3-9f452c025bdd> in <module>()
      2                    target_field='origin',
      3                    model_name='tpu',
----> 4                    tpu_address = tpu_address)

5 frames

/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py in __init__(self, src, **kwds)
   1889         kwds["usecols"] = self.usecols
   1890 
-> 1891         self._reader = parsers.TextReader(src, **kwds)
   1892         self.unnamed_cols = self._reader.unnamed_cols
   1893 

pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader.__cinit__()

pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._setup_parser_source()

FileNotFoundError: [Errno 2] File tpu_train/metadata/results.csv does not exist: 'tpu_train/metadata/results.csv'

from automl_gs import automl_grid_search

automl_grid_search("data.csv", "diagnosis")

Solving a binary_classification problem, maximizing accuracy using tensorflow.

Modeling with field specifications:
id: ignore
radius_mean: numeric
texture_mean: numeric
perimeter_mean: numeric
area_mean: numeric
smoothness_mean: numeric
compactness_mean: numeric
concavity_mean: numeric
concave points_mean: numeric
symmetry_mean: numeric
fractal_dimension_mean: numeric
radius_se: numeric
texture_se: numeric
perimeter_se: numeric
area_se: numeric
smoothness_se: numeric
compactness_se: numeric
concavity_se: numeric
concave points_se: numeric
symmetry_se: numeric
fractal_dimension_se: numeric
radius_worst: numeric
texture_worst: numeric
perimeter_worst: numeric
area_worst: numeric
smoothness_worst: numeric
compactness_worst: numeric
concavity_worst: numeric
concave points_worst: numeric
symmetry_worst: numeric
fractal_dimension_worst: numeric
Unnamed: 32: numeric
0%
0/100 [00:04<?, ?trial/s]
0%
0/20 [00:00<?, ?epoch/s]
---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
<ipython-input-37-308e97508c91> in <module>()
      1 from automl_gs import automl_grid_search
      2 
----> 3 automl_grid_search("data.csv", "diagnosis")

5 frames
/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py in __init__(self, src, **kwds)
   1889         kwds["usecols"] = self.usecols
   1890 
-> 1891         self._reader = parsers.TextReader(src, **kwds)
   1892         self.unnamed_cols = self._reader.unnamed_cols
   1893 

pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader.__cinit__()

pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._setup_parser_source()

FileNotFoundError: [Errno 2] File automl_train/metadata/results.csv does not exist: 'automl_train/metadata/results.csv'

Solving a binary_classification problem, maximizing accuracy using tensorflow.

Modeling with field specifications:
PassengerId: numeric
Pclass: categorical
Name: ignore
Sex: categorical
Age: numeric
SibSp: categorical
Parch: categorical
Ticket: ignore
Fare: numeric
Cabin: categorical
Embarked: categorical

0% 0/100 [00:00<?, ?trial/s]
0% 0/20 [00:00<?, ?epoch/s]

---------------------------------------------------------------------------

FileNotFoundError                         Traceback (most recent call last)

<ipython-input-5-17dc9e2d602c> in <module>()
      2                    target_field='Survived',
      3                    model_name='tpu',
----> 4                    tpu_address = tpu_address)

/usr/local/lib/python3.6/dist-packages/automl_gs/automl_gs.py in automl_grid_search(csv_path, target_field, target_metric, framework, model_name, context, num_trials, split, num_epochs, col_types, gpu, tpu_address)
     85         # and append to the metrics CSV.
     86         results = pd.read_csv(os.path.join(train_folder, 
---> 87                                         "metadata", "results.csv"))
     88         results = results.assign(**params)
     89         results.insert(0, 'trial_id', uuid.uuid4())

/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py in parser_f(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, skipfooter, skip_footer, doublequote, delim_whitespace, as_recarray, compact_ints, use_unsigned, low_memory, buffer_lines, memory_map, float_precision)
    707                     skip_blank_lines=skip_blank_lines)
    708 
--> 709         return _read(filepath_or_buffer, kwds)
    710 
    711     parser_f.__name__ = name

/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py in _read(filepath_or_buffer, kwds)
    447 
    448     # Create the parser.
--> 449     parser = TextFileReader(filepath_or_buffer, **kwds)
    450 
    451     if chunksize or iterator:

/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py in __init__(self, f, engine, **kwds)
    816             self.options['has_index_names'] = kwds['has_index_names']
    817 
--> 818         self._make_engine(self.engine)
    819 
    820     def close(self):

/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py in _make_engine(self, engine)
   1047     def _make_engine(self, engine='c'):
   1048         if engine == 'c':
-> 1049             self._engine = CParserWrapper(self.f, **self.options)
   1050         else:
   1051             if engine == 'python':

/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py in __init__(self, src, **kwds)
   1693         kwds['allow_leading_cols'] = self.index_col is not False
   1694 
-> 1695         self._reader = parsers.TextReader(src, **kwds)
   1696 
   1697         # XXX

pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader.__cinit__()

pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._setup_parser_source()

FileNotFoundError: File b'tpu_train/metadata/results.csv' does not exist