Evaluation on 3DFAW about unsup3d HOT 3 OPEN

import os
from glob import glob
import numpy as np
import shutil
import cv2


split = np.loadtxt('/users/szwu/DepthNets/depthnet-pytorch/data/3DFAW/list_valid_test.txt', dtype=str, delimiter=',')

rootin_im = '/users/szwu/DepthNets/depthnet-pytorch/data/3DFAW/valid_img/'
rootin_kpt = '/users/szwu/DepthNets/depthnet-pytorch/data/3DFAW/valid_lm/'
rootout_ori_im = '/users/szwu/DepthNets/depthnet-pytorch/data/test_depthcorr/ori/img/'
rootout_ori_kpt = '/users/szwu/DepthNets/depthnet-pytorch/data/test_depthcorr/ori/kpts/'
rootout_crop_im = '/users/szwu/DepthNets/depthnet-pytorch/data/test_depthcorr/cropped/img/'
rootout_crop_kpt = '/users/szwu/DepthNets/depthnet-pytorch/data/test_depthcorr/cropped/kpts/'

oriens = ['left','center','right']
for ori in oriens:
    for rootout in [rootout_ori_im, rootout_ori_kpt, rootout_crop_im, rootout_crop_kpt]:
        dirout = os.path.join(rootout, ori)
        if not os.path.isdir(dirout):
            os.makedirs(dirout)

for fid, sp, ori in split:
    if ori in oriens:
        im_fname = fid + '.jpg'
        im_fpath = os.path.join(rootin_im, im_fname)
        shutil.copyfile(im_fpath, os.path.join(rootout_ori_im, ori, im_fname))

        kpt_fname = fid + '_lm.csv'
        kpt_fpath = os.path.join(rootin_kpt, kpt_fname)
        shutil.copyfile(kpt_fpath, os.path.join(rootout_ori_kpt, ori, kpt_fname))

        kpt = np.loadtxt(kpt_fpath, delimiter=',')

        im = cv2.imread(im_fpath)
        a2 = int((kpt.max(0)-kpt.min(0))[:2].mean()*1)
        im = cv2.copyMakeBorder(im,a2,a2,a2,a2,cv2.BORDER_REPLICATE)

        leye = kpt[36:42,:2].mean(0)
        reye = kpt[42:48,:2].mean(0)
        lmouth = kpt[48,:2]
        rmouth = kpt[54,:2]
        nose = kpt[30,:2]

        c1 = (leye + reye + lmouth + rmouth)/4
        c2 = nose - 2*(nose - c1)
        xc, yc = (c2 + 0.5*((leye + reye)/2 - c1)).astype(int)

        x1 = xc-a2 + a2
        x2 = x1+a2*2
        y1 = yc-a2 +int(0.1*a2) + a2
        y2 = y1+a2*2

        crop = im[y1:y2, x1:x2, :]
        crop = cv2.resize(crop, (256,256))
        cv2.imwrite(os.path.join(rootout_crop_im, ori, im_fname), crop)

        s = 256/(a2*2)
        kpt_xy = kpt[:,:2] - np.array([[x1, y1]]) +a2
        kpt_xy = kpt_xy *s
        kpt_z = kpt[:,2:] *s
        np.savetxt(os.path.join(rootout_crop_kpt, ori, kpt_fname), np.concatenate([kpt_xy, kpt_z], 1), delimiter=',')

import cv2
import numpy as np
import torch
import matplotlib.pyplot as plt
import os
from glob import glob


# with border (reported in paper)
result_dir = '/scratch/shared/nfs1/szwu/all_results/finetuned_celeba_test_depthcorr_080_db0.7_crop1.4/'
# result_dir = '/scratch/shared/nfs1/szwu/all_results/3dfaw_test_depthcorr_100_db0.7_crop1.4/'
source_dir = '/scratch/shared/nfs1/szwu/data/test_depthcorr'

kpts_fpaths = sorted(glob(os.path.join(source_dir, 'cropped/kpts/center/*.csv')))
kpts = np.array([np.loadtxt(kpts_fpath, delimiter=',') for kpts_fpath in kpts_fpaths])

crop = 1.4
kpts_xy = kpts[:,:,:2]
s = 256/(64*crop)
kpts_xy = kpts_xy / s
p = 64*(crop-1)/2
kpts_xy = kpts_xy - p
kpts_xy[:,:,1] -= 4

ori_kpts_fpaths = sorted(glob(os.path.join(source_dir, 'ori/kpts/center/*.csv')))
ori_kpts = np.array([np.loadtxt(kpts_fpath, delimiter=',') for kpts_fpath in ori_kpts_fpaths])

im_fpaths = sorted(glob(os.path.join(result_dir, 'input/*.png')))[:75]
depth_fpaths = sorted(glob(os.path.join(result_dir, 'depth/*.png')))[:75]
depths = np.array([cv2.imread(depth_fpath, -1)/65535. for depth_fpath in depth_fpaths])
ori_im_fpaths = sorted(glob(os.path.join(source_dir, 'ori/img/center/*.jpg')))
ori_ims = np.array([cv2.imread(im_fpath)[:,:,::-1] for im_fpath in ori_im_fpaths])


def compute_covar(preds, actuals, n_kps=66):
    return np.sum(np.diag(np.abs(np.corrcoef(preds, actuals, rowvar=0)[:n_kps,n_kps:])))

avg_szs = np.array([sum(im.shape[:2])/2 for im in ori_ims])
gt_z_rescaled = ori_kpts[:,:,2] / avg_szs[:,None]

depths_t = torch.FloatTensor(depths).unsqueeze(1)
kpts_xy_t = torch.FloatTensor(kpts_xy).unsqueeze(1)/32-1

depths_t = torch.nn.functional.pad(depths_t, (32,32,32,32), mode='replicate')
z_sampled = torch.nn.functional.grid_sample(depths_t, kpts_xy_t/2)[:,0,0,:]
z_sampled = z_sampled - z_sampled.mean(1,keepdim=True)

score = compute_covar(z_sampled, gt_z_rescaled)
print(score)