接上篇 人脸关键点检测和头部姿态估计数据集整理
1. 借助于OpenPose生成人脸关键点和头部姿态，使用链接：https://github.com/TadasBaltrusaitis/OpenFace/wiki/Command-line-arguments
build/bin/FaceLandmarkImg.exe -fdir "images path"具体使用可参考博客，每张人脸可得到一个csv文件，里面记录了人脸的很多信息，我们只需要68个关键点和头部姿态
2. 借助于人脸检测网络生成人脸检测结果，每张人脸可得到一个csv文件，里面记录了人脸的左上角和右下角坐标，具体使用可参考博客
3.结合OpenPose和人脸检测结果生成数据集label标注文件，格式为：
face_path x1 y1 x2 y2 x3 y3 ...... x68 y68 pitch yaw roll执行代码：Generate_labels.py
import sys, os import cv2 import numpy as np import csv import argparse import math from shutil import copyfile #Determine whether it is a rotation matrix def isRotationMatrix(R): Rt = np.transpose(R) shouldBeIdentity = np.dot(Rt, R) I = np.identity(3, dtype=R.dtype) n = np.linalg.norm(I - shouldBeIdentity) return n < 1e-6 #rotationvector to rorationmatrix def vec2matrix(rvec): theta = np.linalg.norm(rvec) r = rvec / theta R_ = np.array([[0, -r[2][0], r[1][0]], [r[2][0], 0, -r[0][0]], [-r[1][0], r[0][0], 0]]) R = np.cos(theta) * np.eye(3) + (1 - np.cos(theta)) * r * r.T + np.sin(theta) * R_ return R #rotationmatrix to EulerAngles def rotationMatrixToAngles(R): assert (isRotationMatrix(R)) sy = math.sqrt(R[0, 0] * R[0, 0] + R[1, 0] * R[1, 0]) singular = sy < 1e-6 if not singular: x = math.atan2(R[2, 1], R[2, 2]) y = math.atan2(-R[2, 0], sy) z = math.atan2(R[1, 0], R[0, 0]) else: x = math.atan2(-R[1, 2], R[1, 1]) y = math.atan2(-R[2, 0], sy) z = 0 x = x*180.0/3.141592653589793 y = y*180.0/3.141592653589793 z = z*180.0/3.141592653589793 return np.array([-1.0*x+10, y, z]) def compute_iou(rec1, rec2): """ computing IoU :param rec1: (y0, x0, y1, x1), which reflects (top, left, bottom, right) :param rec2: (y0, x0, y1, x1) :return: scala value of IoU """ # computing area of each rectangles S_rec1 = (rec1[2] - rec1[0]) * (rec1[3] - rec1[1]) S_rec2 = (rec2[2] - rec2[0]) * (rec2[3] - rec2[1]) # computing the sum_area sum_area = S_rec1 + S_rec2 # find the each edge of intersect rectangle left_line = max(rec1[1], rec2[1]) right_line = min(rec1[3], rec2[3]) top_line = max(rec1[0], rec2[0]) bottom_line = min(rec1[2], rec2[2]) # judge if there is an intersect if left_line >= right_line or top_line >= bottom_line: return 0 else: intersect = (right_line - left_line) * (bottom_line - top_line) return (intersect / (sum_area - intersect)) * 1.0 def Generate_labels(img_path, csv_path, txt_path, face_path, label_path): """ :param img_path: :param csv_path: :param txt_path: :param face_path: :param label_path: :return: """ img_list = os.listdir(img_path) fp_label = open(os.path.join(label_path, '300w_label.txt'), 'w') for img_file in img_list: print('now is dealing with '+str(img_file)) # Iterative execution of each image filename, extension = os.path.splitext(img_file) extension = extension[1:] if extension == 'jpg' or extension == 'png': img = cv2.imread(os.path.join(img_path, img_file)) # read and organize the result of OpenFace, result is saved to 'csv_result' for format as: # [ # [point0_x,point0_y], # [point1_x,point1_y], # [point2_x,point2_y], # ...... # [pose_x,pose_y,pose_z], # [xmin_csv,ymin_csv,xmax_csv,ymax_csv] # ] if not os.path.exists(os.path.join(csv_path, filename + '.csv')): continue csv_file = open(os.path.join(csv_path, filename + '.csv'), 'r') reader = csv.reader(csv_file) csv_result = [] for item in reader: csv_result_temp = [] xmin_csv = 50000 ymin_csv = 50000 xmax_csv = 0 ymax_csv = 0 if reader.line_num == 1: continue for i in range(68): csv_result_temp.append([float(item[296 + i]), float(item[296 + i + 68])]) xmin_csv = min(xmin_csv, float(item[296 + i])) xmax_csv = max(xmax_csv, float(item[296 + i])) ymin_csv = min(ymin_csv, float(item[296 + i + 68])) ymax_csv = max(ymax_csv, float(item[296 + i + 68])) csv_result_temp.append([item[293], item[294], item[295]]) csv_result_temp.append([xmin_csv, ymin_csv, xmax_csv, ymax_csv]) csv_result.append(csv_result_temp) csv_file.close() # read and organize the label of 300w dataset, result is saved to 'pts_result' for format as: # [ # [point0_x,point0_y], # [point1_x,point1_y], # [point2_x,point2_y], # ...... # [xmin_pts,ymin_pts,xmax_pts,ymax_pts] # ] fp_pts = open(os.path.join(img_path, filename + '.pts'), 'r') lines = fp_pts.readlines() lines = lines[3:71] xmin_pts = 50000 ymin_pts = 50000 xmax_pts = 0 ymax_pts = 0 pts_result = [] for line in lines: S = line.split(' ') point_x = float(S[0]) point_y = float(S[1]) xmin_pts = min(xmin_pts, point_x) ymin_pts = min(ymin_pts, point_y) xmax_pts = max(xmax_pts, point_x) ymax_pts = max(ymax_pts, point_y) pts_result.append([point_x, point_y]) pts_result.append([xmin_pts, ymin_pts, xmax_pts, ymax_pts]) fp_pts.close() face_pose = [] face_bbox = [] iou_max = 0 for i in range(len(csv_result)): if compute_iou((ymin_pts, xmin_pts, ymax_pts, xmax_pts), ( csv_result[i][69][1], csv_result[i][69][0], csv_result[i][69][3], csv_result[i][69][2])) > iou_max\ and compute_iou((ymin_pts, xmin_pts, ymax_pts, xmax_pts), ( csv_result[i][69][1], csv_result[i][69][0], csv_result[i][69][3], csv_result[i][69][2])) >= 0.5: face_pose = csv_result[i][68][0:] if not face_pose: continue # read the face detected result with txt format and get the face bbox, which is in Square form fp_txt = open(os.path.join(txt_path, filename + '.txt'), 'r') lines = fp_txt.readlines() iou_max = 0 for line in lines: line = line[:-1] S = line.split(' ') xmin_txt = float(S[0]) ymin_txt = float(S[1]) xmax_txt = float(S[2]) ymax_txt = float(S[3]) if compute_iou((ymin_pts, xmin_pts, ymax_pts, xmax_pts), (ymin_txt, xmin_txt, ymax_txt, xmax_txt)) > iou_max\ and compute_iou((ymin_pts, xmin_pts, ymax_pts, xmax_pts), (ymin_txt, xmin_txt, ymax_txt, xmax_txt)) >= 0.7: face_bbox.append([xmin_txt, ymin_txt, xmax_txt, ymax_txt]) fp_txt.close() # determine if the face is detected if face_bbox: width = xmax_txt - xmin_txt height = ymax_txt - ymin_txt adjust_value = 1.0 * (height - width) / 2.0 xmin = int(round(max(0, xmin_txt - adjust_value))) ymin = int(round(ymin_txt)) xmax = int(round(min(img.shape[1], xmin+height))) ymax = int(round(ymax_txt)) print(xmax-xmin, ymax-ymin) else: width = xmax_pts - xmin_pts height = ymax_pts - ymin_pts long_size = max(width, height) adjust_value_x = 1.0 * (long_size - width) / 2.0 adjust_value_y = 1.0 * (long_size - height) / 2.0 xmin = int(round(max(0, xmin_pts - adjust_value_x))) xmax = int(round(min(img.shape[1], xmin+long_size))) ymin = int(round(max(0, ymin_pts - adjust_value_y))) ymax = int(round(min(img.shape[0], ymin+long_size))) print(xmax-xmin,ymax-ymin) # crop face from original img img_face = img[ymin:ymax, xmin:xmax, :] cv2.imwrite(os.path.join(face_path, filename + '.jpg'), img_face) # write the path of cropped face to txt fp_label.write(os.path.join(face_path, filename + '.jpg') + ' ') #fp_label.write('/home/OpenFace/300w_face/'+filename + '.jpg'+' ') # write the point coordinates to txt for i in range(68): fp_label.write(str(pts_result[i][0] - xmin) + ' ' + str(pts_result[i][1] - ymin) + ' ') # write the face pose to txt ''' rotation_vector = [] rotation_vector.append([float(face_pose[0])]) rotation_vector.append([float(face_pose[1])]) rotation_vector.append([float(face_pose[2])]) R = vec2matrix(rotation_vector) headpose = rotationMatrixToAngles(R) ''' headpose = [] headpose.append(float(face_pose[0])/math.pi*180) headpose.append(float(face_pose[1])/math.pi*180) headpose.append(float(face_pose[2])/math.pi*180) fp_label.write(str(headpose[0]) + ' ' + str(headpose[1]) +' ' + str(headpose[2]) + '\n') fp_label.close() if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("--img_path", type=str, default='img_path', help="the path to the 300w dataset") parser.add_argument("--csv_path", type=str, default='csv_path', help="the path to the result of OpenFace") parser.add_argument("--txt_path", type=str, default='txt_path', help="the path to the face detect") parser.add_argument("--face_path", type=str, default='face_path_test', help="the path to the crop face saved") parser.add_argument("--label_path", type=str, default='label_path_test', help="the path to the label of 300w dataset") opt = parser.parse_args() Generate_labels(opt.img_path, opt.csv_path, opt.txt_path, opt.face_path, opt.label_path)4.参考github：https://github.com/lsy17096535/face-landmark/blob/master/train/mainloop.py
生成hdf5时对数据集做的预处理操作如下所示：
image 人脸 1. resize(img,(60,60)).astype('f4') 2. cvtcolor(BGR2GRAY).reshape(1,60,60) 3. m,s = cv2.meanstdDev(image) 4. image = (image - m) / (1.e-6 + s) landmarks 关键点 1. 乘以scale系数 rx=60/face img width ry=60/face img height 2. 归一化 /60 poses 头部姿态 1. 归一化 /50执行代码：Generate_hdf5.py
import sys,os import cv2 import numpy as np from shutil import copyfile caffe_root = 'caffe/python/' sys.path.insert(0, caffe_root + 'python') import caffe import h5py import argparse IMAGE_SIZE = 60 #fixed size to all images def Generate_hdf5(train_txt_file, train_h5_file, train_h5_list_file): """ Generate hdf5 format based on the txt label :param train_txt_file: label in txt format (imgpath x1 y1 x2 y2 ...... x68 y68 pitch yaw roll) :param train_h5_file: h5 file to be saved :param train_h5_list_file: list file in txt format to be saved :return: """ with open(train_txt_file,'r') as T: lines = T.readlines() HD5Images = np.zeros([len(lines), 1, IMAGE_SIZE, IMAGE_SIZE], dtype='float32') HD5Landmarks = np.zeros([len(lines), 136], dtype='float32') HD5Poses = np.zeros([len(lines), 3], dtype='float32') for i,l in enumerate(lines): sp = l.split(' ') print(sp[0]) img = cv2.imread(sp[0]) height,width = img.shape[0], img.shape[1] rx,ry = 1.0*IMAGE_SIZE/width, 1.0*IMAGE_SIZE/height res = cv2.resize(img, (IMAGE_SIZE, IMAGE_SIZE), 0.0, 0.0, interpolation=cv2.INTER_CUBIC) image = res.astype('f4') image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY).reshape(1, IMAGE_SIZE, IMAGE_SIZE) m, s = cv2.meanStdDev(image) image = (image - m) / (1.e-6 + s) HD5Images[i, :] = image label_ = np.zeros([136], dtype='float32') pose_ = np.zeros([3], dtype='float32') for j in range(136): if (j + 1) % 2: scale_factor = rx else: scale_factor = ry label_[j] = float(sp[j + 1]) * float(scale_factor) label_[j] = label_[j]/(1.0*IMAGE_SIZE) label_[j] = label_[j].astype('f4') HD5Landmarks[i, :] = label_[:] # print(HD5Landmarks[i, :]) for j in range(3): normalize_factor = 50 pose_[j] = float(sp[j + 1 + 136]) / float(normalize_factor) pose_[j] = pose_[j].astype('f4') HD5Poses[i, :] = pose_[:] with h5py.File(train_h5_file, 'w') as H: H.create_dataset('data', data=HD5Images) H.create_dataset('label', data=HD5Landmarks) H.create_dataset('pose', data=HD5Poses) with open(train_h5_list_file, 'w') as L: L.write(train_h5_file) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("--train_txt_file",type=str,default="label_path/300w_label.txt", help="path to label in txt format") parser.add_argument("--train_h5_file",type=str,default="test.h5",help="path to generated h5 file") parser.add_argument("--train_h5_list_file",type=str,default="test_h5_list.txt",help="path to generated he list file") opt=parser.parse_args() Generate_hdf5(opt.train_txt_file, opt.train_h5_file, opt.train_h5_list_file)5. 可视化hdf5文件是否正确
执行代码：visual_h5.py
import sys,os import cv2 import h5py import numpy as np f = h5py.File('../data/300w_ori/train.h5','r') data = f['data'] label = f['label'] pose = f['pose'] num = data.shape[0] for i in range(1,100): # print(data.shape) img = np.array(data[i,0,:,:]*255) image = np.zeros([img.shape[0],img.shape[1],3],dtype='float32') image[:,:,0] = img[:] image[:,:,1] = img[:] image[:,:,2] = img[:] print(image.shape) landmark = np.array(label[i,:]) print(landmark) # print(landmark.shape) cv2.imwrite(os.path.join('../data/300w_ori/result',str(i)+'.jpg'),image) img = cv2.imread(os.path.join('../data/300w_ori/result',str(i)+'.jpg')) for j in range(68): x = int(round(landmark[2*j]*60)) y = int(round(landmark[2*j+1]*60)) # print(x,y) cv2.circle(img,(x,y),1,(0,0,255),1) Euler = np.array(pose[i,:]) # print(Euler.shape) cv2.imwrite(os.path.join('../data/300w_ori/result',str(i)+'.jpg'), img)
