前面的都是使用了mediapipe框架。后面的這兩節(jié)采用了opencv\dlib的框架。

一 臉部檢測(cè)

核心：opencv

detectMultiScale函數(shù)

detectMultiScale(image, scaleFactor, minNeighbors, flags, minSize, maxSize)

?image--待檢測(cè)圖片，一般為灰度圖像加快檢測(cè)速度；

scaleFactor參數(shù)控制每個(gè)圖像序列的縮放比例。該參數(shù)決定了在每個(gè)圖像序列中檢測(cè)窗口的大小。默認(rèn)值為1.1，表示每次圖像被縮小10%。較小的值可以捕捉更多的細(xì)節(jié)，但也會(huì)增加計(jì)算量。較大的值可以加快檢測(cè)速度，但可能會(huì)錯(cuò)過(guò)一些目標(biāo)。

minNeighbors參數(shù)定義了每個(gè)目標(biāo)至少應(yīng)該有多少個(gè)鄰居，才能被認(rèn)為是一個(gè)目標(biāo)。該參數(shù)用于過(guò)濾檢測(cè)到的目標(biāo)。

flags參數(shù)用于定義檢測(cè)模式。它可以是以下幾個(gè)值的組合：

• CASCADE_SCALE_IMAGE：使用縮放圖像進(jìn)行檢測(cè)（默認(rèn)值）。
• CASCADE_FIND_BIGGEST_OBJECT：只檢測(cè)最大的目標(biāo)。
• CASCADE_DO_ROUGH_SEARCH：快速搜索模式。

minSize、maxSize參數(shù)用于指定檢測(cè)目標(biāo)的最小、最大尺寸。

?src/yahboom_esp32_mediapipe/yahboom_esp32_mediapipe/目錄下新建05_FaceEyeDetection.py，代碼如下：

#!/usr/bin/env python2
# encoding: utf-8
#import ros lib
import rclpy
from rclpy.node import Node
from geometry_msgs.msg import Point
import mediapipe as mp
from cv_bridge import CvBridge
from sensor_msgs.msg import CompressedImage,Image
#import define msg
from yahboomcar_msgs.msg import PointArray
#import commom lib
import cv2 as cv
import numpy as np
import time
from cv_bridge import CvBridge
from sensor_msgs.msg import Image, CompressedImagefrom rclpy.time import Time
import datetimeprint("import done")class FaceEyeDetection(Node):def __init__(self,name):super().__init__(name)self.bridge = CvBridge()#加載分類器self.eyeDetect = cv.CascadeClassifier( "/home/bohu/yahboomcar/yahboomcar_ws/src/yahboom_esp32_mediapipe/resource/haarcascade_eye.xml")self.faceDetect = cv.CascadeClassifier("/home/bohu/yahboomcar/yahboomcar_ws/src/yahboom_esp32_mediapipe/resource/haarcascade_frontalface_default.xml")self.pub_rgb = self.create_publisher(Image,"/FaceEyeDetection/image", 500)def cancel(self):self.pub_rgb.unregister()def face(self, frame):gray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY)#灰度f(wàn)aces = self.faceDetect.detectMultiScale(gray, 1.3)#檢測(cè)for face in faces: frame = self.faceDraw(frame, face)return framedef eye(self, frame):gray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY)#灰度eyes = self.eyeDetect.detectMultiScale(gray, 1.3)#檢測(cè)for eye in eyes:cv.circle(frame, (int(eye[0] + eye[2] / 2), int(eye[1] + eye[3] / 2)), (int(eye[3] / 2)), (0, 0, 255), 2)return frame#畫框顯示def faceDraw(self, frame, bbox, l=30, t=10):x, y, w, h = bboxx1, y1 = x + w, y + hcv.rectangle(frame, (x, y), (x + w, y + h), (255, 0, 255), 2)# Top left x,ycv.line(frame, (x, y), (x + l, y), (255, 0, 255), t)cv.line(frame, (x, y), (x, y + l), (255, 0, 255), t)# Top right x1,ycv.line(frame, (x1, y), (x1 - l, y), (255, 0, 255), t)cv.line(frame, (x1, y), (x1, y + l), (255, 0, 255), t)# Bottom left x1,y1cv.line(frame, (x, y1), (x + l, y1), (255, 0, 255), t)cv.line(frame, (x, y1), (x, y1 - l), (255, 0, 255), t)# Bottom right x1,y1cv.line(frame, (x1, y1), (x1 - l, y1), (255, 0, 255), t)cv.line(frame, (x1, y1), (x1, y1 - l), (255, 0, 255), t)return framedef pub_img(self, frame):self.pub_rgb.publish(self.bridge.cv2_to_imgmsg(frame, "bgr8"))class MY_Picture(Node):def __init__(self, name):super().__init__(name)self.bridge = CvBridge()self.sub_img = self.create_subscription(CompressedImage, '/espRos/esp32camera', self.handleTopic, 1) #獲取esp32傳來(lái)的圖像self.last_stamp = Noneself.new_seconds = 0self.fps_seconds = 1self.face_eye_detection = FaceEyeDetection('face_eye_detection')self.content = ["face", "eye", "face_eye"]self.content_index = 0#回調(diào)函數(shù)def handleTopic(self, msg):self.last_stamp = msg.header.stamp  if self.last_stamp:total_secs = Time(nanoseconds=self.last_stamp.nanosec, seconds=self.last_stamp.sec).nanosecondsdelta = datetime.timedelta(seconds=total_secs * 1e-9)seconds = delta.total_seconds()*100if self.new_seconds != 0:self.fps_seconds = seconds - self.new_secondsself.new_seconds = seconds#保留這次的值start = time.time()frame = self.bridge.compressed_imgmsg_to_cv2(msg)frame = cv.resize(frame, (640, 480))cv.waitKey(10)action = cv.waitKey(1) & 0xFFif action == ord("f") or action == ord("F"):self.content_index += 1if self.content_index >= len(self.content): self.content_index = 0if self.content[self.content_index] == "face": frame = self.face_eye_detection.face(frame)elif self.content[self.content_index] == "eye": frame = self.face_eye_detection.eye(frame)else: frame = self.face_eye_detection.eye(self.face_eye_detection.face(frame))end = time.time()fps = 1 / ((end - start)+self.fps_seconds)text = "FPS : " + str(int(fps))cv.putText(frame, text, (20, 30), cv.FONT_HERSHEY_SIMPLEX, 0.9, (0, 0, 255), 1)cv.imshow('frame', frame)self.face_eye_detection.pub_img(frame)# print(frame)cv.waitKey(10)def main():print("start it")rclpy.init()esp_img = MY_Picture("My_Picture")try:rclpy.spin(esp_img)except KeyboardInterrupt:passfinally:esp_img.destroy_node()rclpy.shutdown()

主要流程跟之前類似，這個(gè)不如mediapipe框架好，尤其是畫框比較麻煩。

這個(gè)識(shí)別效果對(duì)比之前的face_recognition，我覺得不如那個(gè)好，參見：

ros2-4.2 用python實(shí)現(xiàn)人臉識(shí)別_ros2使用人臉檢測(cè)-CSDN博客

構(gòu)建后運(yùn)行：

ros2 run yahboom_esp32_mediapipe FaceEyeDetection

效果如下

?

二 人臉特效

人臉檢測(cè)

使用了dlib庫(kù)

.get_frontal_face_detector()

功能：人臉檢測(cè)畫框
參數(shù)：無(wú)
返回值：默認(rèn)的人臉檢測(cè)器

?shape_predictor()

功能：標(biāo)記人臉關(guān)鍵點(diǎn)
參數(shù)：shape_predictor_68_face_landmarks.dat：68個(gè)關(guān)鍵點(diǎn)模型地址
返回值：人臉關(guān)鍵點(diǎn)預(yù)測(cè)器

import cv2
import mediapipe as mp
import dlibdetector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor("/home/bohu/yahboomcar/yahboomcar_ws/src/yahboom_esp32_mediapipe/resource/shape_predictor_68_face_landmarks.dat")cap = cv2.VideoCapture(0)#打開默認(rèn)攝像頭
while True:ret,frame = cap.read()#讀取一幀圖像#圖像格式轉(zhuǎn)換frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)# 因?yàn)閿z像頭是鏡像的，所以將攝像頭水平翻轉(zhuǎn)# 不是鏡像的可以不翻轉(zhuǎn)frame= cv2.flip(frame,1)#輸出結(jié)果gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)faces = detector(gray)print(f'faces:{len(faces)}')for face in faces:# 利用預(yù)測(cè)器預(yù)測(cè)shape = predictor(gray, face)# print(shape)# 標(biāo)出68個(gè)點(diǎn)的位置for i in range(68):cv2.circle(frame, (shape.part(i).x, shape.part(i).y), 2, (0, 255, 0), -1, 1)cv2.putText(frame, str(i), (shape.part(i).x, shape.part(i).y), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255))cv2.imshow('opencv detectMultiScale', frame)if cv2.waitKey(1) & 0xFF == 27:break
cap.release()

人臉特效

基本思路就是在dlib人臉檢測(cè)點(diǎn)上，在額外使用opencv畫線\和fillConvexPoly填充多邊形函數(shù)。

這種就是比較麻煩，沒有官方的函數(shù)之間調(diào)用好。

src/yahboom_esp32_mediapipe/yahboom_esp32_mediapipe/目錄下新建文件06_FaceLandmarks.py，代碼如下：

#!/usr/bin/env python3
# encoding: utf-8
import rclpy
from rclpy.node import Node
import time
import dlib
import cv2 as cv
import numpy as np
from cv_bridge import CvBridge
from sensor_msgs.msg import Image, CompressedImagefrom rclpy.time import Time
import datetimeclass FaceLandmarks:def __init__(self, dat_file):self.hog_face_detector = dlib.get_frontal_face_detector()self.dlib_facelandmark = dlib.shape_predictor(dat_file)def get_face(self, frame, draw=True):gray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY)self.faces = self.hog_face_detector(gray)for face in self.faces:self.face_landmarks = self.dlib_facelandmark(gray, face)if draw:for n in range(68):x = self.face_landmarks.part(n).xy = self.face_landmarks.part(n).ycv.circle(frame, (x, y), 2, (0, 255, 255), 2)cv.putText(frame, str(n), (x, y), cv.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 255), 2)return framedef get_lmList(self, frame, p1, p2, draw=True):lmList = []if len(self.faces) != 0:for n in range(p1, p2):x = self.face_landmarks.part(n).xy = self.face_landmarks.part(n).ylmList.append([x, y])if draw:next_point = n + 1if n == p2 - 1: next_point = p1x2 = self.face_landmarks.part(next_point).xy2 = self.face_landmarks.part(next_point).ycv.line(frame, (x, y), (x2, y2), (0, 255, 0), 1)return lmListdef get_lipList(self, frame, lipIndexlist, draw=True):lmList = []if len(self.faces) != 0:for n in range(len(lipIndexlist)):x = self.face_landmarks.part(lipIndexlist[n]).xy = self.face_landmarks.part(lipIndexlist[n]).ylmList.append([x, y])if draw:next_point = n + 1if n == len(lipIndexlist) - 1: next_point = 0x2 = self.face_landmarks.part(lipIndexlist[next_point]).xy2 = self.face_landmarks.part(lipIndexlist[next_point]).ycv.line(frame, (x, y), (x2, y2), (0, 255, 0), 1)return lmListdef prettify_face(self, frame, eye=True, lips=True, eyebrow=True, draw=True):if eye:leftEye = landmarks.get_lmList(frame, 36, 42)rightEye = landmarks.get_lmList(frame, 42, 48)if draw:if len(leftEye) != 0: frame = cv.fillConvexPoly(frame, np.mat(leftEye), (0, 0, 0))if len(rightEye) != 0: frame = cv.fillConvexPoly(frame, np.mat(rightEye), (0, 0, 0))if lips:lipIndexlistA = [51, 52, 53, 54, 64, 63, 62]lipIndexlistB = [48, 49, 50, 51, 62, 61, 60]lipsUpA = landmarks.get_lipList(frame, lipIndexlistA, draw=True)lipsUpB = landmarks.get_lipList(frame, lipIndexlistB, draw=True)lipIndexlistA = [57, 58, 59, 48, 67, 66]lipIndexlistB = [54, 55, 56, 57, 66, 65, 64]lipsDownA = landmarks.get_lipList(frame, lipIndexlistA, draw=True)lipsDownB = landmarks.get_lipList(frame, lipIndexlistB, draw=True)if draw:if len(lipsUpA) != 0: frame = cv.fillConvexPoly(frame, np.mat(lipsUpA), (249, 0, 226))if len(lipsUpB) != 0: frame = cv.fillConvexPoly(frame, np.mat(lipsUpB), (249, 0, 226))if len(lipsDownA) != 0: frame = cv.fillConvexPoly(frame, np.mat(lipsDownA), (249, 0, 226))if len(lipsDownB) != 0: frame = cv.fillConvexPoly(frame, np.mat(lipsDownB), (249, 0, 226))if eyebrow:lefteyebrow = landmarks.get_lmList(frame, 17, 22)righteyebrow = landmarks.get_lmList(frame, 22, 27)if draw:if len(lefteyebrow) != 0: frame = cv.fillConvexPoly(frame, np.mat(lefteyebrow), (255, 255, 255))if len(righteyebrow) != 0: frame = cv.fillConvexPoly(frame, np.mat(righteyebrow), (255, 255, 255))return frameclass MY_Picture(Node):def __init__(self, name,landmarkss):super().__init__(name)self.bridge = CvBridge()self.sub_img = self.create_subscription(CompressedImage, '/espRos/esp32camera', self.handleTopic, 1) #獲取esp32傳來(lái)的圖像self.landmarksros = landmarkssself.last_stamp = Noneself.new_seconds = 0self.fps_seconds = 1def handleTopic(self, msg):self.last_stamp = msg.header.stamp  if self.last_stamp:total_secs = Time(nanoseconds=self.last_stamp.nanosec, seconds=self.last_stamp.sec).nanosecondsdelta = datetime.timedelta(seconds=total_secs * 1e-9)seconds = delta.total_seconds()*100if self.new_seconds != 0:self.fps_seconds = seconds - self.new_secondsself.new_seconds = seconds#保留這次的值start = time.time()frame = self.bridge.compressed_imgmsg_to_cv2(msg)frame = cv.resize(frame, (640, 480))cv.waitKey(10)frame = self.landmarksros.get_face(frame, draw=False)frame = self.landmarksros.prettify_face(frame, eye=True, lips=True, eyebrow=True, draw=True)end = time.time()fps = 1 / ((end - start)+self.fps_seconds)text = "FPS : " + str(int(fps))cv.putText(frame, text, (20, 30), cv.FONT_HERSHEY_SIMPLEX, 0.9, (0, 0, 255), 1)cv.imshow('frame', frame)landmarks = None
def main():global landmarksprint("start it")#使用官方訓(xùn)練好的dlib 68點(diǎn)模型dat_file = "/home/bohu/yahboomcar/yahboomcar_ws/src/yahboom_esp32_mediapipe/resource/shape_predictor_68_face_landmarks.dat"landmarks = FaceLandmarks(dat_file)rclpy.init()esp_img = MY_Picture("My_Picture",landmarks)try:rclpy.spin(esp_img)except KeyboardInterrupt:passfinally:esp_img.destroy_node()rclpy.shutdown()

構(gòu)建后運(yùn)行：

?ros2 run yahboom_esp32_mediapipe FaceLandmarks

效果如下：