shen/Intelligent-Elderly-Care
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

funtion integraty

Browse Source
This commit is contained in:
Tang1705
2020-07-10 22:58:44 +08:00
parent aaa3799fe2
commit b79ae2e115
18 changed files with 4283 additions and 337 deletions
Download Patch File Download Diff File Expand all files Collapse all files

203
Class/collect_data/face_reco_from_camera.py
View File

@@ -1,4 +1,5 @@
# 摄像头实时人脸识别
import threading
import dlib
import numpy as np
@@ -9,8 +10,11 @@ import time
import facenet
from PIL import Image, ImageDraw, ImageFont
from Post import post
from model import create_model
import smile_detection
start_time = 0
# 1. Dlib 正向人脸检测器
# detector = dlib.get_frontal_face_detector()
@@ -20,10 +24,10 @@ detector = cv2.dnn.readNetFromCaffe("data/data_opencv/deploy.prototxt.txt",
"data/data_opencv/res10_300x300_ssd_iter_140000.caffemodel")
# 2. Dlib 人脸 landmark 特征点检测器
predictor = dlib.shape_predictor('data/data_dlib/shape_predictor_68_face_landmarks.dat')
# predictor = dlib.shape_predictor('data/data_dlib/shape_predictor_68_face_landmarks.dat')
# 3. Dlib Resnet 人脸识别模型，提取 128D 的特征矢量
face_reco_model = dlib.face_recognition_model_v1("data/data_dlib/dlib_face_recognition_resnet_model_v1.dat")
# face_reco_model = dlib.face_recognition_model_v1("data/data_dlib/dlib_face_recognition_resnet_model_v1.dat")
nn4_small2_pretrained = create_model()
nn4_small2_pretrained.load_weights('weights/nn4.small2.v1.h5')
@@ -35,6 +39,7 @@ class Face_Recognizer:
self.features_known_list = []
# 存储录入人脸名字
self.loaded = False
self.name_known_cnt = 0
self.name_known_list = []
@@ -44,6 +49,7 @@ class Face_Recognizer:
# 存储当前摄像头中捕获到的所有人脸的坐标名字
self.pos_camera_list = []
self.name_camera_list = []
self.type_camera_list = []
# 存储当前摄像头中捕获到的人脸数
self.faces_cnt = 0
# 存储当前摄像头中捕获到的人脸特征
@@ -55,31 +61,38 @@ class Face_Recognizer:
# 从 "features_all.csv" 读取录入人脸特征
def get_face_database(self):
if os.path.exists("data/data_faces_from_camera/"):
self.metadata = facenet.load_metadata("data/data_faces_from_camera/")
self.name_known_cnt = self.metadata.shape[0]
self.embedded = np.zeros((self.metadata.shape[0], 128))
for i, m in enumerate(self.metadata):
for j, n in enumerate(m):
img = facenet.load_image(n.image_path())
# img = align_image(img)
img = cv2.resize(img, (96, 96))
# scale RGB values to interval [0,1]
img = (img / 255.).astype(np.float32)
# obtain embedding vector for image
self.embedded[i] = nn4_small2_pretrained.predict(np.expand_dims(img, axis=0))[0]
# self.embedded[i] = self.embedded[i] / len(m)
self.name_known_list.append('')
if self.loaded:
return 1
else:
print('##### Warning #####', '\n')
print("'features_all.csv' not found!")
print(
"Please run 'get_faces_from_camera.py' before 'face_reco_from_camera.py'",
'\n')
print('##### End Warning #####')
return 0
if os.path.exists("data/data_faces_from_camera/"):
self.metadata = facenet.load_metadata("data/data_faces_from_camera/")
self.name_known_cnt = self.metadata.shape[0]
self.embedded = np.zeros((self.metadata.shape[0], 128))
for i, m in enumerate(self.metadata):
for j, n in enumerate(m):
for k, p in enumerate(n):
img = facenet.load_image(p.image_path())
# img = align_image(img)
img = cv2.resize(img, (96, 96))
# scale RGB values to interval [0,1]
img = (img / 255.).astype(np.float32)
# obtain embedding vector for image
self.embedded[i] = nn4_small2_pretrained.predict(np.expand_dims(img, axis=0))[0]
# self.embedded[i] = self.embedded[i] / len(m)
path = p.image_path().replace("\\", "/")
self.name_known_list.append(path.split('/')[-2])
self.type_camera_list.append(path.split('/')[-3])
self.loaded = True
return 1
else:
print('##### Warning #####', '\n')
print("'features_all.csv' not found!")
print(
"Please run 'get_faces_from_camera.py' before 'face_reco_from_camera.py'",
'\n')
print('##### End Warning #####')
return 0
# 计算两个128D向量间的欧式距离
# @staticmethod
@@ -106,23 +119,25 @@ class Face_Recognizer:
def draw_name(self, img_rd):
# 在人脸框下面写人脸名字
img_with_name = img_rd
font = ImageFont.truetype("simsun.ttc", 30, index=1)
img = Image.fromarray(cv2.cvtColor(img_rd, cv2.COLOR_BGR2RGB))
draw = ImageDraw.Draw(img)
for i in range(self.faces_cnt):
# cv2.putText(img_rd, self.name_camera_list[i], self.pos_camera_list[i], font, 0.8, (0, 255, 255), 1, cv2.LINE_AA)
draw.text(xy=self.pos_camera_list[i], text=self.name_camera_list[i], font=font)
img_with_name = cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
if self.name_camera_list[i] != 'unknown':
# cv2.putText(img_rd, self.name_camera_list[i], self.pos_camera_list[i], font, 0.8, (0, 255, 255), 1, cv2.LINE_AA)
draw.text(xy=self.pos_camera_list[i], text=self.name_camera_list[i], font=font)
img_with_name = cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
return img_with_name
# 修改显示人名
def modify_name_camera_list(self):
# TODO 数据库 ID
# Default known name: person_1, person_2, person_3
self.name_known_list[0] = '唐麒'.encode('utf-8').decode()
self.name_known_list[1] = '段海燕'.encode('utf-8').decode()
# Default known name: 1, 2, person_3
self.name_known_list[0] = '1'.encode('utf-8').decode()
self.name_known_list[1] = 'Tony Blair'.encode('utf-8').decode()
# self.name_known_list[2] = '唐保生'.encode('utf-8').decode()
# self.name_known_list[3] = '唐麒'.encode('utf-8').decode()
# self.name_known_list[3] = '1'.encode('utf-8').decode()
# self.name_known_list[4] ='xx'.encode('utf-8').decode()
# 处理获取的视频流，进行人脸识别
@@ -180,7 +195,7 @@ class Face_Recognizer:
# 确定人名的位置坐标
# 先默认所有人不认识，是 unknown
# Set the default names of faces with "unknown"
self.name_camera_list.append("陌生人")
self.name_camera_list.append("unknown")
# 每个捕获人脸的名字坐标
box = faces[0, 0, k, 3:7] * np.array([w, h, w, h])
@@ -188,62 +203,84 @@ class Face_Recognizer:
self.pos_camera_list.append(tuple(
[int(startX + 5), int(startY - 30)]))
height = (endY - startY)
width = (endX - startX)
# height = (endY - startY)
# width = (endX - startX)
img_blank = np.zeros((height, width, 3), np.uint8)
for ii in range(height):
for jj in range(width):
img_blank[ii][jj] = img_rd[startY + ii][startX + jj]
# img_blank = np.zeros((height, width, 3), np.uint8)
img_blank = img_rd[startY:endY, startX:endX]
try:
# for ii in range(height):
# for jj in range(width):
# img_blank[ii][jj] = img_rd[startY + ii][startX + jj]
img = cv2.resize(img_blank, (96, 96))
img = (img / 255.).astype(np.float32)
img = nn4_small2_pretrained.predict(np.expand_dims(img, axis=0))[0]
img = cv2.resize(img_blank, (96, 96))
img = (img / 255.).astype(np.float32)
img = nn4_small2_pretrained.predict(np.expand_dims(img, axis=0))[0]
# 5. 对于某张人脸，遍历所有存储的人脸特征
e_distance_list = []
for i in range(0, len(self.embedded)):
e_distance_list.append(facenet.distance(self.embedded[i], img))
# for i in range(len(self.features_known_list)):
# # 如果 person_X 数据不为空
# if str(self.features_known_list[i][0]) != '0.0':
# # print("with person", str(i + 1), "the e distance: ", end='')
# e_distance_tmp = self.return_euclidean_distance(self.features_camera_list[k],
# self.features_known_list[i])
# # print(e_distance_tmp)
# e_distance_list.append(e_distance_tmp)
# else:
# # 空数据 person_X
# e_distance_list.append(999999999)
# # 6. 寻找出最小的欧式距离匹配
similar_person_num = e_distance_list.index(min(e_distance_list))
# print("Minimum e distance with person", self.name_known_list[similar_person_num])
# print(min(e_distance_list))
if min(e_distance_list) < 0.58:
self.name_camera_list[k] = self.name_known_list[similar_person_num % 8]
# print("May be person " + str(self.name_known_list[similar_person_num]))
else:
pass
# print("Unknown person")
# 矩形框
for kk, d in enumerate(faces):
# 绘制矩形框
if self.name_camera_list[k] != '陌生人':
cv2.rectangle(img_rd, tuple([startX, startY]), tuple([endX, endY]),
(0, 255, 0), 2)
cv2.rectangle(img_rd, tuple([startX, startY - 35]), tuple([endX, startY]),
(0, 255, 0), cv2.FILLED)
# 5. 对于某张人脸，遍历所有存储的人脸特征
e_distance_list = []
for i in range(0, len(self.embedded)):
e_distance_list.append(facenet.distance(self.embedded[i], img))
# for i in range(len(self.features_known_list)):
# # 如果 person_X 数据不为空
# if str(self.features_known_list[i][0]) != '0.0':
# # print("with person", str(i + 1), "the e distance: ", end='')
# e_distance_tmp = self.return_euclidean_distance(self.features_camera_list[k],
# self.features_known_list[i])
# # print(e_distance_tmp)
# e_distance_list.append(e_distance_tmp)
# else:
# # 空数据 person_X
# e_distance_list.append(999999999)
# # 6. 寻找出最小的欧式距离匹配
similar_person_num = e_distance_list.index(min(e_distance_list))
# print("Minimum e distance with person", self.name_known_list[similar_person_num])
# print(min(e_distance_list))
if min(e_distance_list) < 0.58:
self.name_camera_list[k] = self.name_known_list[similar_person_num % 8]
if self.type_camera_list[similar_person_num % 8] == 'elder':
mode = smile_detection.smile_detect(img_blank)
if mode == 'happy':
cv2.imwrite('smile_detection.jpg', img_rd)
cv2.rectangle(img_rd, tuple([startX, startY - 70]),
tuple([endX, startY - 35]),
(0, 215, 255), cv2.FILLED)
cv2.putText(img_rd, 'happy', (startX + 5, startY - 45), cv2.FONT_ITALIC, 1,
(255, 255, 255), 1, cv2.LINE_AA)
# t = threading.Thread(target=post(elder_id=self.name_camera_list[k], event=0,
# imagePath='smile_detection.jpg'))
# t.start()
# print("May be person " + str(self.name_known_list[similar_person_num]))
elif min(e_distance_list) > 0.75:
self.name_camera_list[k] = '陌生人'
cv2.imwrite('stranger_detection.jpg', img_rd)
# t = threading.Thread(target=post(event=2, imagePath='stranger_detection.jpg'))
# t.start()
else:
cv2.rectangle(img_rd, tuple([startX, startY]), tuple([endX, endY]),
(0, 0, 255), 2)
cv2.rectangle(img_rd, tuple([startX, startY - 35]), tuple([endX, startY]),
(0, 0, 255), cv2.FILLED)
pass
# print("Unknown person")
# 矩形框
for kk, d in enumerate(faces):
# 绘制矩形框
if self.name_camera_list[k] == '陌生人':
cv2.rectangle(img_rd, tuple([startX, startY]), tuple([endX, endY]),
(0, 0, 255), 2)
cv2.rectangle(img_rd, tuple([startX, startY - 35]), tuple([endX, startY]),
(0, 0, 255), cv2.FILLED)
elif self.name_camera_list[k] != 'unknown':
cv2.rectangle(img_rd, tuple([startX, startY]), tuple([endX, endY]),
(0, 255, 0), 2)
cv2.rectangle(img_rd, tuple([startX, startY - 35]), tuple([endX, startY]),
(0, 255, 0), cv2.FILLED)
except:
continue
# print('\n')
# self.faces_cnt = faces.shape[2]
# if len(self.name_camera_list) > 0:
# 7. 在这里更改显示的人名
self.modify_name_camera_list()
# self.modify_name_camera_list()
# 8. 写名字
# self.draw_name(img_rd)
img_with_name = self.draw_name(img_rd)
@@ -269,6 +306,10 @@ class Face_Recognizer:
def main():
# Calibration_on = Calibration()
# scale = Calibration_on.run()
# print(scale)
Face_Recognizer_con = Face_Recognizer()
Face_Recognizer_con.run()

53
Class/collect_data/get_faces_from_camera.py
View File

@@ -17,6 +17,7 @@ action_map = {'look_ahead': '请看前方', 'blink': '请眨眼', 'open_mouth':
'smile': '请笑一笑', 'rise_head': '请抬头',
'bow_head': '请低头', 'look_left': '请看左边',
'look_right': '请看右边', 'over': '录入完成'}
people_type_dict = {'0': 'elder', '1': 'worker', '2': 'volunteer'}
# Dlib 正向人脸检测器
# detector = dlib.get_frontal_face_detector()
@@ -27,7 +28,7 @@ detector = cv2.dnn.readNetFromCaffe("data/data_opencv/deploy.prototxt.txt",
class Face_Register:
def __init__(self):
def __init__(self, people_type, id):
self.init = False
self.path_photos_from_camera = "data/data_faces_from_camera/"
self.font = cv2.FONT_ITALIC
@@ -47,18 +48,24 @@ class Face_Register:
self.frame_start_time = 0
self.fps = 0
def speak(self, text, rate=2):
self.people_type = people_type_dict[str(people_type)]
self.id = id
def speak(self):
text = action_map[action_list[self.index]]
speak = win32com.client.Dispatch('Sapi.SpVoice')
# speak.Voice = speak.GetVoices('Microsoft Zira')
speak.Volume = 100
speak.Rate = rate
speak.Rate = 2
speak.Speak(text)
# 新建保存人脸图像文件
def pre_work_mkdir(self):
if os.path.isdir(self.path_photos_from_camera):
pass
if os.path.isdir(self.path_photos_from_camera + '/' + self.people_type):
pass
else:
os.mkdir(self.path_photos_from_camera + '/' + self.people_type)
else:
os.mkdir(self.path_photos_from_camera)
@@ -71,7 +78,8 @@ class Face_Register:
if os.path.isfile("data/features_all.csv"):
os.remove("data/features_all.csv")
# 如果有之前录入的人脸, 在之前 person_x 的序号按照 person_x+1 开始录入
# 如果有之前录入的人脸, 在之前 person_x 的序号按照 person_x+1 开始录入 (v1)
# 根据传入的人员类型和 id 录入 (v2)
def check_existing_faces_cnt(self):
if os.listdir("data/data_faces_from_camera/"):
# 获取已录入的最后一个人脸序号
@@ -81,8 +89,8 @@ class Face_Register:
person_num_list.append(int(person.split('_')[-1]))
self.existing_faces_cnt = max(person_num_list)
# 如果第一次存储或者没有之前录入的人脸, 按照 person_1 开始录入
# Start from person_1
# 如果第一次存储或者没有之前录入的人脸, 按照 1 开始录入
# Start from 1
else:
self.existing_faces_cnt = 0
@@ -127,7 +135,7 @@ class Face_Register:
# self.pre_work_del_old_face_folders()
# 3. 检查 "/data/data_faces_from_camera" 中已有人脸文件
self.check_existing_faces_cnt()
# self.check_existing_faces_cnt()
while stream.isOpened():
self.faces_cnt = 0
@@ -143,8 +151,9 @@ class Face_Register:
# 4. 按下 'n' 新建存储人脸的文件夹
if kk == ord('n'):
self.existing_faces_cnt += 1
current_face_dir = self.path_photos_from_camera + "person_" + str(self.existing_faces_cnt)
# self.existing_faces_cnt += 1
current_face_dir = self.path_photos_from_camera + '/' + self.people_type + '/' + self.id
# current_face_dir = self.path_photos_from_camera + "person_" + str(self.existing_faces_cnt)
os.makedirs(current_face_dir)
print('\n')
print("新建的人脸文件夹 / Create folders: ", current_face_dir)
@@ -174,12 +183,13 @@ class Face_Register:
height = (endY - startY)
width = (endX - startX)
hh = int(height / 2)
ww = int(width / 2)
# hh = int(height / 2)
# ww = int(width / 2)
# 6. 判断人脸矩形框是否超出 480x640
if (endX + ww) > 640 or (endY + hh > 480) or (startX - ww < 0) or (
startY - hh < 0):
if endX > 640 or endY > 480or startX < 0 or startY < 0:
# if (endX + ww) > 640 or (endY + hh > 480) or (startX - ww < 0) or (
# startY - hh < 0):
cv2.putText(img_rd, "OUT OF RANGE", (20, 300), self.font, 0.8, (0, 0, 255), 1, cv2.LINE_AA)
color_rectangle = (0, 0, 255)
save_flag = 0
@@ -190,8 +200,8 @@ class Face_Register:
save_flag = 1
cv2.rectangle(img_rd,
tuple([startX - ww, startY - hh]),
tuple([endX + ww, endY + hh]),
tuple([startX , startY]),
tuple([endX , endY ]),
color_rectangle, 2)
# 7. 根据人脸大小生成空的图像
@@ -208,12 +218,13 @@ class Face_Register:
if self.index <= 7:
for ii in range(height):
for jj in range(width):
img_blank[ii][jj] = img_rd[startY + ii][startX + jj]
img_blank[ii][jj] = img_rd[startY + ii][startX + jj]
cv2.imwrite(current_face_dir + "/img_face_" + str(self.ss_cnt) + ".jpg", img_blank)
print("写入本地 / Save into：",
str(current_face_dir) + "/img_face_" + str(self.ss_cnt) + ".jpg")
self.index += 1
self.speak(action_map[action_list[self.index]], 2)
if self.index < len(action_list) - 1:
self.index += 1
self.speak()
else:
print("请先按 'N' 来建文件夹, 按 'S' / Please press 'N' and press 'S'")
# self.faces_cnt = len(faces)
@@ -231,7 +242,7 @@ class Face_Register:
cv2.imshow("camera", img_rd)
if not self.init:
self.speak(self.speak(action_map[action_list[self.index]], 2))
self.speak()
self.init = True
def run(self):
@@ -243,7 +254,7 @@ class Face_Register:
def main():
Face_Register_con = Face_Register()
Face_Register_con = Face_Register(people_type=1, id='3')
Face_Register_con.run()