Hi, I'm trying to use adahessian in TensorFlow for a simple regression experiment but

Help using adahessian in TensorFlow about adahessian HOT 3 OPEN

amirgholami commented on May 30, 2024

Help using adahessian in TensorFlow

from adahessian.

Comments (3)

KimiHsieh commented on May 30, 2024

I have the same issue

from adahessian.

KimiHsieh commented on May 30, 2024

Environment: adahessian_tf/environment.yml

I think the issue is caused by grads = gradients.gradients(loss, params) in get_gradients_hessian(self, loss, params)
if you check the return of grads = gradients.gradients(loss, params), it will be None.
But I don't know how to fix this issue.

adahessian/adahessian_tf/cifar_training_tools.py

Lines 56 to 71 in 935a047

 @tf.function 

 def step(x, y, training): 

 with tf.GradientTape() as tape: 

 r_loss = tf.add_n(model.losses) 

 outs = model(x, training) 

 c_loss = loss_fn(y, outs) 

 loss = c_loss + r_loss 

 if training: 

 if optim_method != 'adahessian': 

 gradients = tape.gradient(loss, model.trainable_weights) 

 optimizer.apply_gradients(zip(gradients, model.trainable_weights)) 

 else: 

 gradients, Hessian = optimizer.get_gradients_hessian(loss, model.trainable_weights) 

 optimizer.apply_gradients_hessian(zip(gradients, Hessian, model.trainable_weights))

from adahessian.

KimiHsieh commented on May 30, 2024

Environment: adahessian_tf/environment.yml

I think the issue is caused by grads = gradients.gradients(loss, params) in get_gradients_hessian(self, loss, params)
if you check the return of grads = gradients.gradients(loss, params), it will be None.
But I don't know how to fix this issue.

adahessian/adahessian_tf/cifar_training_tools.py

Lines 56 to 71 in 935a047

@tf.function

def step(x, y, training):

with tf.GradientTape() as tape:

r_loss = tf.add_n(model.losses)

outs = model(x, training)

c_loss = loss_fn(y, outs)

loss = c_loss + r_loss

if training:

if optim_method != 'adahessian':

gradients = tape.gradient(loss, model.trainable_weights)

optimizer.apply_gradients(zip(gradients, model.trainable_weights))

else:

gradients, Hessian = optimizer.get_gradients_hessian(loss, model.trainable_weights)

optimizer.apply_gradients_hessian(zip(gradients, Hessian, model.trainable_weights))

adahessian/adahessian_tf/adahessian.py

Lines 160 to 198 in 935a047

 def get_gradients_hessian(self, loss, params): 

 """Returns gradients and Hessian of `loss` with respect to `params`. 

  Arguments: 

  loss: Loss tensor. 

  params: List of variables. 

  Returns: 

  List of gradient and Hessian tensors. 

  Raises: 

  ValueError: In case any gradient cannot be computed (e.g. if gradient 

  function not implemented). 

  """ 

 params = nest.flatten(params) 

 with backend.get_graph().as_default(), backend.name_scope(self._name + 

 "/gradients"): 

 grads = gradients.gradients(loss, params) 

 for grad, param in zip(grads, params): 

 if grad is None: 

 raise ValueError("Variable {} has `None` for gradient. " 

 "Please make sure that all of your ops have a " 

 "gradient defined (i.e. are differentiable). " 

 "Common ops without gradient: " 

 "K.argmax, K.round, K.eval.".format(param)) 

 # WARNING: for now we do not support gradient clip 

 # grads = self._clip_gradients(grads) 

 v = [np.random.uniform(0, 1, size = p.shape) for p in params] 

 for vi in v: 

 vi[ vi < 0.5] = -1 

 vi[ vi >= 0.5] = 1 

 v = [tf.convert_to_tensor(vi, dtype = tf.dtypes.float32) for vi in v] 

 vprod = tf.reduce_sum([ tf.reduce_sum(vi * grad) for vi, grad in zip(v, grads)]) 

 Hv = gradients.gradients(vprod, params) 

 Hd = [ tf.abs(Hvi * vi) for Hvi, vi in zip(Hv, v)] 

 return grads, Hd

from adahessian.

Help using adahessian in TensorFlow about adahessian HOT 3 OPEN

Comments (3)

Related Issues (20)

Recommend Projects

React

Vue.js

Typescript

TensorFlow

Django

Laravel

D3

Recommend Topics

javascript

web

server

Machine learning

Visualization

Game

Recommend Org

Facebook

Microsoft

Google

Alibaba

D3

Tencent

	@tf.function
	def step(x, y, training):
	with tf.GradientTape() as tape:
	r_loss = tf.add_n(model.losses)
	outs = model(x, training)
	c_loss = loss_fn(y, outs)
	loss = c_loss + r_loss

	if training:

	if optim_method != 'adahessian':
	gradients = tape.gradient(loss, model.trainable_weights)
	optimizer.apply_gradients(zip(gradients, model.trainable_weights))
	else:
	gradients, Hessian = optimizer.get_gradients_hessian(loss, model.trainable_weights)
	optimizer.apply_gradients_hessian(zip(gradients, Hessian, model.trainable_weights))

	def get_gradients_hessian(self, loss, params):
	"""Returns gradients and Hessian of `loss` with respect to `params`.
	Arguments:
	loss: Loss tensor.
	params: List of variables.
	Returns:
	List of gradient and Hessian tensors.
	Raises:
	ValueError: In case any gradient cannot be computed (e.g. if gradient
	function not implemented).
	"""
	params = nest.flatten(params)
	with backend.get_graph().as_default(), backend.name_scope(self._name +
	"/gradients"):
	grads = gradients.gradients(loss, params)
	for grad, param in zip(grads, params):
	if grad is None:
	raise ValueError("Variable {} has `None` for gradient. "
	"Please make sure that all of your ops have a "
	"gradient defined (i.e. are differentiable). "
	"Common ops without gradient: "
	"K.argmax, K.round, K.eval.".format(param))

	# WARNING: for now we do not support gradient clip
	# grads = self._clip_gradients(grads)

	v = [np.random.uniform(0, 1, size = p.shape) for p in params]
	for vi in v:
	vi[ vi < 0.5] = -1
	vi[ vi >= 0.5] = 1
	v = [tf.convert_to_tensor(vi, dtype = tf.dtypes.float32) for vi in v]

	vprod = tf.reduce_sum([ tf.reduce_sum(vi * grad) for vi, grad in zip(v, grads)])

	Hv = gradients.gradients(vprod, params)

	Hd = [ tf.abs(Hvi * vi) for Hvi, vi in zip(Hv, v)]

	return grads, Hd