INFO:runner:epoch 0/0: running for 100000 batches of 1:
Traceback (most recent call last):
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 1322, in _do_call
return fn(*args)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 1307, in _run_fn
options, feed_dict, fetch_list, target_list, run_metadata)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 1409, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.InvalidArgumentError: filter must be 4-dimensional: [1,7,7,16,32]
[[Node: train12_errorrepr/1/mdgru/dim1/backward/rnn/while/CGRUCell/Gates/ConvLinear/convolution_1/Conv2D = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:GPU:0"](train12_errorrepr/1/mdgru/dim1/backward/rnn/while/CGRUCell/Gates/ConvLinear/convolution_1/ExpandDims, train12_errorrepr/1/mdgru/dim1/backward/rnn/while/CGRUCell/Gates/ConvLinear/convolution_1/ExpandDims_1)]]
[[Node: train12_errorrepr/split_2/_93 = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_14829_train12_errorrepr/split_2", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "RUN_mdgru.py", line 460, in
Runner(ev, experiments_postfix="_" + pf, **args_runner).run()
File "/home/antal/Programs/tensorflow-mdgru-seminew/runner/init.py", line 346, in run
self.train()
File "/home/antal/Programs/tensorflow-mdgru-seminew/runner/init.py", line 234, in train
loss = self.ev.train(batch_size=self.batch_size)
File "/home/antal/Programs/tensorflow-mdgru-seminew/eval/init.py", line 247, in train
_, loss, summary = self.sess.run(tasks, ph)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 900, in run
run_metadata_ptr)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 1135, in _run
feed_dict_tensor, options, run_metadata)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 1316, in _do_run
run_metadata)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 1335, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: filter must be 4-dimensional: [1,7,7,16,32]
[[Node: train12_errorrepr/1/mdgru/dim1/backward/rnn/while/CGRUCell/Gates/ConvLinear/convolution_1/Conv2D = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:GPU:0"](train12_errorrepr/1/mdgru/dim1/backward/rnn/while/CGRUCell/Gates/ConvLinear/convolution_1/ExpandDims, train12_errorrepr/1/mdgru/dim1/backward/rnn/while/CGRUCell/Gates/ConvLinear/convolution_1/ExpandDims_1)]]
[[Node: train12_errorrepr/split_2/_93 = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_14829_train12_errorrepr/split_2", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]]
Caused by op 'train12_errorrepr/1/mdgru/dim1/backward/rnn/while/CGRUCell/Gates/ConvLinear/convolution_1/Conv2D', defined at:
File "RUN_mdgru.py", line 453, in
args_eval)
File "/home/antal/Programs/tensorflow-mdgru-seminew/eval/init.py", line 326, in init
super(LargeVolumeEvaluation, self).init(model, collectioninst, kw)
File "/home/antal/Programs/tensorflow-mdgru-seminew/eval/classification.py", line 12, in init
kw)
File "/home/antal/Programs/tensorflow-mdgru-seminew/eval/init.py", line 205, in init
self.model.optimize
File "/home/antal/Programs/tensorflow-mdgru-seminew/model/init.py", line 19, in wrapper
setattr(self, attribute, function(self))
File "/home/antal/Programs/tensorflow-mdgru-seminew/model/mdgru_classification.py", line 82, in optimize
rets = optimizer.minimize(self.cost, global_step=self.global_step)
File "/home/antal/Programs/tensorflow-mdgru-seminew/model/init.py", line 19, in wrapper
setattr(self, attribute, function(self))
File "/home/antal/Programs/tensorflow-mdgru-seminew/model/init.py", line 110, in cost
loss = tf.reduce_mean(self.costs)
File "/home/antal/Programs/tensorflow-mdgru-seminew/model/init.py", line 19, in wrapper
setattr(self, attribute, function(self))
File "/home/antal/Programs/tensorflow-mdgru-seminew/model/mdgru_classification.py", line 69, in costs
loss = tf.nn.softmax_cross_entropy_with_logits(logits=self.logits,
File "/home/antal/Programs/tensorflow-mdgru-seminew/model/init.py", line 19, in wrapper
setattr(self, attribute, function(self))
File "/home/antal/Programs/tensorflow-mdgru-seminew/model/mdgru_classification.py", line 46, in logits
h = self.mdgru_bb(h, self.dropout, mdgruc, fcc, name="{}".format(it + 1), istraining=self.training, **kw)
File "/home/antal/Programs/tensorflow-mdgru-seminew/model/mdrnn/init.py", line 59, in mdgru_bb
mdgru = mdgruclass()
File "/home/antal/Programs/tensorflow-mdgru-seminew/model/mdrnn/mdgru.py", line 125, in call
strides=copy(st))
File "/home/antal/Programs/tensorflow-mdgru-seminew/model/mdrnn/mdgru.py", line 174, in add_cgru
swap_memory=self.swap_memory, initial_state=initial_state)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/rnn.py", line 627, in dynamic_rnn
dtype=dtype)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/rnn.py", line 824, in _dynamic_rnn_loop
swap_memory=swap_memory)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/control_flow_ops.py", line 3224, in while_loop
result = loop_context.BuildLoop(cond, body, loop_vars, shape_invariants)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/control_flow_ops.py", line 2956, in BuildLoop
pred, body, original_loop_vars, loop_vars, shape_invariants)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/control_flow_ops.py", line 2893, in _BuildLoop
body_result = body(*packed_vars_for_body)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/control_flow_ops.py", line 3194, in
body = lambda i, lv: (i + 1, orig_body(*lv))
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/rnn.py", line 795, in _time_step
(output, new_state) = call_cell()
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/rnn.py", line 781, in
call_cell = lambda: cell(input_t, state)
File "/home/antal/Programs/tensorflow-mdgru-seminew/model/crnn/cgru.py", line 81, in call
strides=self.strides, orthogonal_init=False)
File "/home/antal/Programs/tensorflow-mdgru-seminew/model/crnn/init.py", line 166, in _convlinear
is_circular_convolution=self.periodic_convolution_h)
File "/home/antal/Programs/tensorflow-mdgru-seminew/model/crnn/init.py", line 106, in _convolution
return convolution_helper_padding_same(data, convolution_filter, filter_shape, strides)
File "/home/antal/Programs/tensorflow-mdgru-seminew/helper.py", line 99, in convolution_helper_padding_same
return tf.nn.convolution(inp, convolution_filter, "SAME", strides)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/nn_ops.py", line 780, in convolution
return op(input, filter)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/nn_ops.py", line 868, in call
return self.conv_op(inp, filter)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/nn_ops.py", line 520, in call
return self.call(inp, filter)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/nn_ops.py", line 204, in call
name=self.name)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/nn_ops.py", line 193, in _conv1d
name=name)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/util/deprecation.py", line 497, in new_func
return func(*args, **kwargs)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/util/deprecation.py", line 497, in new_func
return func(*args, **kwargs)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/nn_ops.py", line 2464, in conv1d
data_format=data_format)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/gen_nn_ops.py", line 956, in conv2d
data_format=data_format, dilations=dilations, name=name)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/ops.py", line 3392, in create_op
op_def=op_def)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/ops.py", line 1718, in init
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
InvalidArgumentError (see above for traceback): filter must be 4-dimensional: [1,7,7,16,32]
[[Node: train12_errorrepr/1/mdgru/dim1/backward/rnn/while/CGRUCell/Gates/ConvLinear/convolution_1/Conv2D = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:GPU:0"](train12_errorrepr/1/mdgru/dim1/backward/rnn/while/CGRUCell/Gates/ConvLinear/convolution_1/ExpandDims, train12_errorrepr/1/mdgru/dim1/backward/rnn/while/CGRUCell/Gates/ConvLinear/convolution_1/ExpandDims_1)]]
[[Node: train12_errorrepr/split_2/_93 = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_14829_train12_errorrepr/split_2", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]]