Modify hand tracking, add right click and scrolling

This commit is contained in:
Piotr Radzewicz 2020-02-14 14:32:36 +01:00
parent b8809db927
commit b09e5fc6f9

346
main.py Normal file → Executable file
View File

@ -1,66 +1,294 @@
import cv2
import numpy
import math
import numpy as np
import pyautogui
import pynput.mouse
def openCamera(input):
cv2.imshow("frame", input)
key = cv2.waitKey(1)
return key
class FingersNumberDetector:
def drawBox(finder, frame, b, g, r):
for x, y, w, h in finder:
cv2.rectangle(frame, (x, y), (x + w, y + h) ,(b, g, r), 2)
def __init__(self):
pyautogui.PAUSE = 0
def exit(video):
video.release()
cv2.destroyAllWindows()
self.isHandHistCreated = False
self.isBgCaptured = False
self.bgSubThreshold = 30
self.dx = 0
self.dy = 0
self.mouse = pynput.mouse.Controller()
# Background subtractor learning rate
self.bgSubtractorLr = 0
self.xs = [6.0/20.0, 9.0/20.0, 12.0/20.0]
self.ys = [9.0/20.0, 10.0/20.0, 11.0/20.0]
# Gamma correction lookUpTable
# Increase the contrast
gamma = 3
self.lookUpTable = np.empty((1, 256), np.uint8)
for i in range(256):
self.lookUpTable[0, i] = np.clip(
pow(i / 255.0, gamma) * 255.0, 0, 255)
def createHandHistogram(self, frame):
rows, cols, _ = frame.shape
hsvFrame = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
roi = np.zeros([180, 20, 3], dtype=hsvFrame.dtype)
i = 0
for x in self.xs:
for y in self.ys:
x0, y0 = int(x*rows), int(y*cols)
roi[i*20:i*20 + 20, :, :] = hsvFrame[x0:x0 + 20, y0:y0 + 20, :]
i += 1
handHist = cv2.calcHist([roi], [0, 1], None, [
180, 256], [0, 180, 0, 256])
return cv2.normalize(handHist, handHist, 0, 255, cv2.NORM_MINMAX)
def drawRect(self, frame):
rows, cols, _ = frame.shape
for x in self.xs:
for y in self.ys:
x0, y0 = int(x*rows), int(y*cols)
cv2.rectangle(frame, (y0, x0),
(y0 + 20, x0 + 20), (0, 255, 0), 1)
def histMasking(self, frame, handHist):
"""Create the HSV masking"""
hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
dst = cv2.calcBackProject([hsv], [0, 1], handHist, [0, 180, 0, 256], 1)
disc = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (21, 21))
cv2.filter2D(dst, -1, disc, dst)
ret, thresh = cv2.threshold(dst, 150, 255, cv2.THRESH_BINARY)
kernel = np.ones((5, 5), np.uint8)
thresh = cv2.morphologyEx(
thresh, cv2.MORPH_CLOSE, kernel, iterations=7)
# thresh = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel, iterations=5)
# thresh = cv2.dilate(thresh, kernel, iterations=5)
# thresh = cv2.erode(thresh, kernel, iterations=5)
thresh = cv2.merge((thresh, thresh, thresh))
return cv2.bitwise_and(frame, thresh)
def getCentroid(self, contour):
moment = cv2.moments(contour)
if moment['m00'] != 0:
cx = int(moment['m10']/moment['m00'])
cy = int(moment['m01']/moment['m00'])
return [cx, cy]
else:
return None
def getMaxContours(self, contours):
maxIndex = 0
maxArea = 0
for i in range(len(contours)):
cnt = contours[i]
area = cv2.contourArea(cnt)
if area > maxArea:
maxArea = area
maxIndex = i
return contours[maxIndex]
def threshold(self, mask):
grayMask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(grayMask, 0, 255, 0)
return thresh
def bgSubMasking(self, frame):
"""Create a foreground (hand) mask
@param frame: The video frame
@return: A masked frame
"""
fgmask = self.bgSubtractor.apply(
frame, learningRate=self.bgSubtractorLr)
kernel = np.ones((4, 4), np.uint8)
# MORPH_OPEN removes noise
# MORPH_CLOSE closes the holes in the object
fgmask = cv2.morphologyEx(fgmask, cv2.MORPH_OPEN, kernel, iterations=2)
fgmask = cv2.morphologyEx(
fgmask, cv2.MORPH_CLOSE, kernel, iterations=2)
return cv2.bitwise_and(frame, frame, mask=fgmask)
def getMaskAreaRatio(self, mask):
grayMask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(grayMask, 0, 255, 0)
return np.sum(thresh)/(self.height*self.width*255)
def setupFrame(self, frame_width, frame_height):
"""self.x0 and self.y0 are top left corner coordinates
self.width and self.height are the width and height the ROI
"""
x, y = 0.0, 0.4
self.x0 = int(frame_width*x)
self.y0 = int(frame_height*y)
self.width = 450
self.height = 450
def countFingers(self, contour, contourAndHull):
hull = cv2.convexHull(contour, returnPoints=False)
if len(hull) > 3:
defects = cv2.convexityDefects(contour, hull)
cnt = 0
if type(defects) != type(None):
for i in range(defects.shape[0]):
s, e, f, d = defects[i, 0]
start = tuple(contour[s, 0])
end = tuple(contour[e, 0])
far = tuple(contour[f, 0])
angle = self.calculateAngle(far, start, end)
# Ignore the defects which are small and wide
# Probably not fingers
if d > 10000 and angle <= math.pi/2:
cnt += 1
cv2.circle(contourAndHull, far, 8, [255, 0, 0], -1)
return True, cnt
return False, 0
def calculateAngle(self, far, start, end):
"""Cosine rule"""
a = math.sqrt((end[0] - start[0])**2 + (end[1] - start[1])**2)
b = math.sqrt((far[0] - start[0])**2 + (far[1] - start[1])**2)
c = math.sqrt((end[0] - far[0])**2 + (end[1] - far[1])**2)
angle = math.acos((b**2 + c**2 - a**2) / (2*b*c))
return angle
def execute(self, cnt):
if cnt == 1:
self.mouse.click(pynput.mouse.Button.left, 1)
if cnt == 2:
self.mouse.click(pynput.mouse.Button.right, 1)
if cnt == 3:
pyautogui.press("down")
elif cnt == 4:
pyautogui.press("up")
def detectHand(self, frame, handHist):
roi = frame[self.y0:self.y0 + self.height,
self.x0:self.x0 + self.width,
:]
roi = cv2.bilateralFilter(roi, 5, 50, 100)
# Color masking
histMask = self.histMasking(roi, handHist)
cv2.imshow("histMask", histMask)
# Background substraction
bgSubMask = self.bgSubMasking(roi)
cv2.imshow("bgSubMask", bgSubMask)
# Attempt to learn the background automatically
"""
areaRatio = self.getMaskAreaRatio(bgSubMask)
if areaRatio > 0.6:
self.bgSubtractorLr = 1
elif areaRatio < 0.001:
self.bgSubtractorLr = 0
"""
# Overall mask
mask = cv2.bitwise_and(histMask, bgSubMask)
thresh = self.threshold(mask)
cv2.imshow("Overall thresh", thresh)
contours, hierarchy = cv2.findContours(
thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
if len(contours) > 0:
maxContour = self.getMaxContours(contours)
M = cv2.moments(maxContour)
if M["m00"] != 0:
cX = int(M["m10"] / M["m00"])
cY = int(M["m01"] / M["m00"])
if self.dx != 0:
self.mouse.move( -1 * 2 * (cX -self.dx), 2 * (cY - self.dy))
self.dx = cX
self.dy = cY
# print(cX, cY)
# Draw contour and hull
contourAndHull = np.zeros(roi.shape, np.uint8)
hull = cv2.convexHull(maxContour)
# print(type(hull))
cv2.drawContours(contourAndHull, [maxContour], 0, (0, 255, 0), 2)
cv2.drawContours(contourAndHull, [hull], 0, (0, 0, 255), 3)
found, cnt = self.countFingers(maxContour, contourAndHull)
cv2.imshow("Contour and Hull", contourAndHull)
if found:
self.execute(cnt)
centroid = self.getCentroid(maxContour)
if centroid is not None:
centroid[0] += self.x0
centroid[1] += self.y0
cv2.circle(frame, tuple(centroid), 5, [255, 0, 0], -1)
def startDetecting(self):
cap = cv2.VideoCapture(0)
frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
self.setupFrame(frame_width, frame_height)
while cap.isOpened():
ret, frame = cap.read()
# Increase the contrast
# frame = cv2.convertScaleAbs(frame, alpha=3, beta=-500)
# Gamma corection
# Increase the contrast
frame = cv2.LUT(frame, self.lookUpTable)
cv2.rectangle(frame, (self.x0, self.y0), (self.x0 +
self.width - 1, self.y0 + self.height - 1), (255, 0, 0), 2)
k = cv2.waitKey(1) & 0xFF
if k == ord("z"):
self.isHandHistCreated = True
handHist = self.createHandHistogram(frame)
elif k == ord('b'):
self.bgSubtractor = cv2.createBackgroundSubtractorMOG2(
30, self.bgSubThreshold)
self.isBgCaptured = True
elif k == ord("r"):
self.bgSubtractor = None
self.isBgCaptured = False
if self.isHandHistCreated and self.isBgCaptured:
self.detectHand(frame, handHist)
elif not self.isHandHistCreated:
self.drawRect(frame)
cv2.imshow("Output", frame)
if k == ord("q"):
break
elif k == ord("j"):
self.y0 = min(self.y0 + 20, frame_height - self.height)
elif k == ord("k"):
self.y0 = max(self.y0 - 20, 0)
elif k == ord("h"):
self.x0 = max(self.x0 - 20, 0)
elif k == ord("l"):
self.x0 = min(self.x0 + 20, frame_width - self.width)
cap.release()
cv2.destroyAllWindows()
mouse = pynput.mouse.Controller()
video = cv2.VideoCapture(0)
frame = video.read()
findFistInit = None
centerX = 0
centerY = 0
while True:
cascadeHand = cv2.CascadeClassifier("hand.xml")
cascadeFist = cv2.CascadeClassifier("fist.xml")
frame = video.read()[1]
fmask = cv2.GaussianBlur(cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY), (21,21), 0)
cv2.imshow("fmask", fmask)
findFist = cascadeFist.detectMultiScale(fmask, scaleFactor = 1.1, minNeighbors = 75)
if findFistInit is None:
findFistInit = findFist
if findFist == ():
findHand = cascadeHand.detectMultiScale(fmask, scaleFactor = 1.1, minNeighbors = 10)
if findHand != ():
mouse.click(pynput.mouse.Button.left, 1)
if findFist != () and findFistInit != ():
dx, dy, dw, dh = findFist[0][0] - findFistInit[0][0], findFist[0][1] - findFistInit[0][1], findFist[0][2] - findFistInit[0][2], findFist[0][3] - findFistInit[0][3]
centerX = (dx + dx + dw) / 2
centerY = (dy + dy + dh) / 2
mouse.move( -1 * 7 * dx, 7 * dy)
findFistInit = findFist
drawBox(findFist, frame, 0, 0, 255)
drawBox(findHand, frame, 0, 255, 0)
frame = cv2.flip(frame, 1)
key = openCamera(frame)
if(key == ord('q')):
break
exit(video)
if __name__ == "__main__":
detector = FingersNumberDetector()
detector.startDetecting()