# zeroing output layer
    nn.init.constant(self.out.weight.data, 0.)
    nn.init.constant(self.out.bias.data, 1.)`

ret = 0      # episode return
r_lst = []   # store reward
p_lst = []   # store price
P_lst = []   # store position
buffer = []

s = env.reset()
done = False
while not done:

    # sample action
    A_Pr = model.forward(Variable(s))
    #print('model:', A_Pr.data)
    act = torch.multinomial(A_Pr.data, num_samples=1)
    i_act = act[0][0]
    #print('act:', i_act)

    # apply action
    s_, r, done = env.step(i_act)

    # tracker
    ret += r
    r_lst.append(r)
    p_lst.append(env.curr_OHLCV()[3])
    P_lst.append(i_act)

    # Save transitions
    buffer.append((s, act, r))

    if done: break

    # Swap states
    s = s_

# Learning when episode finishes
model.train()

S, A, R = zip(*buffer)
del buffer[:]

S = Variable(torch.cat(S))
A = Variable(torch.cat(A))

# Compute target
Q = []
ret = 0
for r in reversed(R):
    ret = r + .9 * ret
    Q.append(ret)
Q.reverse()

# standardize Q
Q = np.array(Q).astype(np.float32)
Q -= Q.mean()
Q /= Q.std() + 1e-6
Q.clip(min=-10, max=10)
Q = np.expand_dims(Q, axis=1)
Q = Variable(torch.from_numpy(Q))

# PG update
if train_mode: #modified.................add a judge
    A_Pr = model.forward(S).gather(1, A).clamp(min=1e-7, max=1 - 1e-7)

    loss = -(Q * torch.log(A_Pr)).mean()
    model.optim.zero_grad()
    loss.backward()
    model.optim.step()

model.eval()
return ret, r_lst, p_lst, P_lst`