From 16d2079189088d633a52261b26dc8da46a5ee370 Mon Sep 17 00:00:00 2001 From: Saketh Chintapalli Date: Tue, 14 Aug 2018 12:14:51 +0530 Subject: [PATCH] Updated emotion_predictor.py to run on Pi Ported the emotion predictor onto the raspberry pi 3B+ to capture a face every 20 seconds and detect the emotion. --- emotion_predictor.py | 111 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 89 insertions(+), 22 deletions(-) diff --git a/emotion_predictor.py b/emotion_predictor.py index b5d5751..3095df4 100644 --- a/emotion_predictor.py +++ b/emotion_predictor.py @@ -3,51 +3,118 @@ from dlib import get_frontal_face_detector, shape_predictor import cv2 import numpy as np +import time +from picamera import PiCamera +from picamera import PiCamera data = {} -prediction_data = [] prediction_labels = [] -clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8)) +clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8)) #create histogram equalization parameters detector = get_frontal_face_detector() -predictor = shape_predictor("/home/deeplearning/Desktop/shape_predictor_68_face_landmarks.dat") -clf = pickle.load(open("/home/deeplearning/PycharmProjects/project2/model.xml", "rb")) +predictor = shape_predictor("/home/pi/Downloads/shape_predictor_68_face_landmarks.dat") #file to get 68 landmarks +clf = pickle.load(open("/home/pi/Downloads/model.sav", "rb")) emotions = ["happy", "anger", "fear", "sadness", "disgust", "surprise", "neutral"] +prediction_image = [] -def get_landmarks(image): +def get_face(image): #function to crop face from image + gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + faceDet = cv2.CascadeClassifier("/home/pi/Downloads/haarcascade_frontalface_default.xml") #Haar classifiers to detect face + faceDet2 = cv2.CascadeClassifier("/home/pi/Downloads/haarcascade_frontalface_alt2.xml") + faceDet3 = cv2.CascadeClassifier("/home/pi/Downloads/haarcascade_frontalface_alt.xml") + faceDet4 = cv2.CascadeClassifier("/home/pi/Downloads/haarcascade_frontalface_alt_tree.xml") + face = faceDet.detectMultiScale(gray, scaleFactor=1.1, minNeighbors=10, minSize=(5, 5), flags=cv2.CASCADE_SCALE_IMAGE) + face2 = faceDet2.detectMultiScale(gray, scaleFactor=1.1, minNeighbors=10, minSize=(5, 5), flags=cv2.CASCADE_SCALE_IMAGE) + face3 = faceDet3.detectMultiScale(gray, scaleFactor=1.1, minNeighbors=10, minSize=(5, 5), flags=cv2.CASCADE_SCALE_IMAGE) + face4 = faceDet4.detectMultiScale(gray, scaleFactor=1.1, minNeighbors=10, minSize=(5, 5), flags=cv2.CASCADE_SCALE_IMAGE) + + if len(face) == 1: + facefeatures = face + elif len(face2) == 1: + facefeatures = face2 + elif len(face3) == 1: + facefeatures = face3 + elif len(face4) == 1: + facefeatures = face4 + for (x, y, w, h) in facefeatures: # Get coordinates and size of rectangle containing face + gray = gray[y:y+h, x:x+w] # Cut the frame to size + try: + out = cv2.resize(gray, (350, 350)) #resize images to standard size + clahe_image = clahe.apply(out) #applying histogram equalization + except: + pass + return clahe_image + + +def get_landmarks(image): #function to produce features from facial landmarks detected = detector(image, 1) for k, d in enumerate(detected): shape = predictor(image, d) xlist = [] ylist = [] for i in range(1,68): - xlist.append(shape.part(i).x) + #cv2.circle(image, (shape.part(i).x, shape.part(i).y), 1, (0, 0, 255), thickness=2) + xlist.append(shape.part(i).x) #appending x and y coordinates of facial landmarks ylist.append(shape.part(i).y) + #cv2.imshow("got landmarks", image) + #cv2.waitKey(3000) xmean = np.mean(xlist) ymean = np.mean(ylist) xcentral = [(x-xmean) for x in xlist] ycentral = [(y-ymean) for y in ylist] vectorized_landmarks = [] - for x,y,w,z in zip(xcentral, ycentral, xlist, ylist): + for x, y, w, z in zip(xcentral, ycentral, xlist, ylist): vectorized_landmarks.append(w) vectorized_landmarks.append(z) mean_array = np.asarray((ymean,xmean)) coordinates_array = np.asarray((z,w)) - normalized_distance = np.linalg.norm(coordinates_array - mean_array) + normalized_distance = np.linalg.norm(coordinates_array - mean_array) #distance from center to other landmarks vectorized_landmarks.append(normalized_distance) - vectorized_landmarks.append((math.atan2(y, x)*360)/(2*math.pi)) + vectorized_landmarks.append((math.atan2(y, x)*360)/(2*math.pi)) #calculating face offset if len(detected) < 1: data["landmark_vectorization"] = "error" - data["landmark_vectorization"] = vectorized_landmarks - - -prediction_image = cv2.imread("/home/deeplearning/Desktop/face_recognition examples/biden.jpg") #input your image -gray = cv2.cvtColor(prediction_image, cv2.COLOR_BGR2GRAY) -clahe_image = clahe.apply(gray) -get_landmarks(clahe_image) -if data["landmark_vectorization"] == "error": - print("no faces detected") -else: - prediction_data.append(data["landmark_vectorization"]) -nparray = np.array(prediction_data) -print(clf.predict_proba(nparray)) + data["landmark_vectorization"] = vectorized_landmarks #appending coordinates(model features) to dictionary + return data + + + +def get_emotion(data, count): #function to use facial features and run it through the ML model + prediction_data = [] + if data["landmark_vectorization"] == "error": + print("no faces detected") + else: + prediction_data.append(data["landmark_vectorization"]) + nparray = np.array(prediction_data) + final_results = clf.predict_proba(nparray)[0] #getting probabilities of each emotion in the image + print(final_results) + font = cv2.FONT_HERSHEY_SIMPLEX + initial_point = 100 + for a,b in zip(emotions, final_results): + confidence_text = str(str(a) + " : " + str(b * 100) + " %") + cv2.putText(prediction_image, confidence_text, (30, initial_point), font, 1, (255,0,0), 1, cv2.LINE_AA) #Writing results on the image + initial_point = initial_point + 50 + final_text = str("Emotion : " + str(emotions[int(np.argmax(final_results))])) + cv2.putText(prediction_image, final_text, (30, initial_point), font, 1, (255,0,0), 1, cv2.LINE_AA) #Writing final emotion prediction to image + cv2.imwrite(str("/home/pi/Desktop/results/image_emotion" + str(count) + ".jpg"), prediction_image) #saving image with results locally + + + +count = 1 +camera = PiCamera() +while True: + camera.start_preview() + time.sleep(10) #sleep for 10 seconds before picture is taken + image_path = str("/home/pi/Desktop/results/image" + str(count) + ".jpg") + camera.capture(image_path) + camera.stop_preview() + prediction_image = cv2.imread(image_path) + clahe_image = get_face(prediction_image) + data = get_landmarks(clahe_image) #getting features for image + get_emotion(data, count) + count = count + 1 + if count == 11: + break + else: + time.sleep(10) #wait for 10 seconds to capture next emotion image + continue +print("images and emotion results saved to local machine.")