Modify hand tracking, add right click and scrolling
This commit is contained in:
Piotr Radzewicz
parent
b8809db927
commit
b09e5fc6f9
346
main.py
Normal file → Executable file
346
main.py
Normal file → Executable file
@ -1,66 +1,294 @@
|
||||
import cv2
|
||||
import numpy
|
||||
import math
|
||||
import numpy as np
|
||||
import pyautogui
|
||||
import pynput.mouse
|
||||
|
||||
def openCamera(input):
|
||||
cv2.imshow("frame", input)
|
||||
key = cv2.waitKey(1)
|
||||
return key
|
||||
class FingersNumberDetector:
|
||||
|
||||
def drawBox(finder, frame, b, g, r):
|
||||
for x, y, w, h in finder:
|
||||
cv2.rectangle(frame, (x, y), (x + w, y + h) ,(b, g, r), 2)
|
||||
def __init__(self):
|
||||
pyautogui.PAUSE = 0
|
||||
|
||||
def exit(video):
|
||||
video.release()
|
||||
cv2.destroyAllWindows()
|
||||
self.isHandHistCreated = False
|
||||
self.isBgCaptured = False
|
||||
self.bgSubThreshold = 30
|
||||
self.dx = 0
|
||||
self.dy = 0
|
||||
self.mouse = pynput.mouse.Controller()
|
||||
|
||||
# Background subtractor learning rate
|
||||
self.bgSubtractorLr = 0
|
||||
|
||||
self.xs = [6.0/20.0, 9.0/20.0, 12.0/20.0]
|
||||
self.ys = [9.0/20.0, 10.0/20.0, 11.0/20.0]
|
||||
|
||||
# Gamma correction lookUpTable
|
||||
# Increase the contrast
|
||||
gamma = 3
|
||||
self.lookUpTable = np.empty((1, 256), np.uint8)
|
||||
for i in range(256):
|
||||
self.lookUpTable[0, i] = np.clip(
|
||||
pow(i / 255.0, gamma) * 255.0, 0, 255)
|
||||
|
||||
def createHandHistogram(self, frame):
|
||||
rows, cols, _ = frame.shape
|
||||
hsvFrame = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
|
||||
roi = np.zeros([180, 20, 3], dtype=hsvFrame.dtype)
|
||||
|
||||
i = 0
|
||||
for x in self.xs:
|
||||
for y in self.ys:
|
||||
x0, y0 = int(x*rows), int(y*cols)
|
||||
roi[i*20:i*20 + 20, :, :] = hsvFrame[x0:x0 + 20, y0:y0 + 20, :]
|
||||
|
||||
i += 1
|
||||
handHist = cv2.calcHist([roi], [0, 1], None, [
|
||||
180, 256], [0, 180, 0, 256])
|
||||
return cv2.normalize(handHist, handHist, 0, 255, cv2.NORM_MINMAX)
|
||||
|
||||
def drawRect(self, frame):
|
||||
rows, cols, _ = frame.shape
|
||||
|
||||
for x in self.xs:
|
||||
for y in self.ys:
|
||||
x0, y0 = int(x*rows), int(y*cols)
|
||||
cv2.rectangle(frame, (y0, x0),
|
||||
(y0 + 20, x0 + 20), (0, 255, 0), 1)
|
||||
|
||||
def histMasking(self, frame, handHist):
|
||||
"""Create the HSV masking"""
|
||||
hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
|
||||
dst = cv2.calcBackProject([hsv], [0, 1], handHist, [0, 180, 0, 256], 1)
|
||||
|
||||
disc = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (21, 21))
|
||||
cv2.filter2D(dst, -1, disc, dst)
|
||||
|
||||
ret, thresh = cv2.threshold(dst, 150, 255, cv2.THRESH_BINARY)
|
||||
|
||||
kernel = np.ones((5, 5), np.uint8)
|
||||
thresh = cv2.morphologyEx(
|
||||
thresh, cv2.MORPH_CLOSE, kernel, iterations=7)
|
||||
# thresh = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel, iterations=5)
|
||||
# thresh = cv2.dilate(thresh, kernel, iterations=5)
|
||||
# thresh = cv2.erode(thresh, kernel, iterations=5)
|
||||
|
||||
thresh = cv2.merge((thresh, thresh, thresh))
|
||||
return cv2.bitwise_and(frame, thresh)
|
||||
|
||||
def getCentroid(self, contour):
|
||||
moment = cv2.moments(contour)
|
||||
if moment['m00'] != 0:
|
||||
cx = int(moment['m10']/moment['m00'])
|
||||
cy = int(moment['m01']/moment['m00'])
|
||||
return [cx, cy]
|
||||
else:
|
||||
return None
|
||||
|
||||
def getMaxContours(self, contours):
|
||||
maxIndex = 0
|
||||
maxArea = 0
|
||||
|
||||
for i in range(len(contours)):
|
||||
cnt = contours[i]
|
||||
area = cv2.contourArea(cnt)
|
||||
|
||||
if area > maxArea:
|
||||
maxArea = area
|
||||
maxIndex = i
|
||||
return contours[maxIndex]
|
||||
|
||||
def threshold(self, mask):
|
||||
grayMask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY)
|
||||
ret, thresh = cv2.threshold(grayMask, 0, 255, 0)
|
||||
return thresh
|
||||
|
||||
def bgSubMasking(self, frame):
|
||||
"""Create a foreground (hand) mask
|
||||
@param frame: The video frame
|
||||
@return: A masked frame
|
||||
"""
|
||||
fgmask = self.bgSubtractor.apply(
|
||||
frame, learningRate=self.bgSubtractorLr)
|
||||
|
||||
kernel = np.ones((4, 4), np.uint8)
|
||||
# MORPH_OPEN removes noise
|
||||
# MORPH_CLOSE closes the holes in the object
|
||||
fgmask = cv2.morphologyEx(fgmask, cv2.MORPH_OPEN, kernel, iterations=2)
|
||||
fgmask = cv2.morphologyEx(
|
||||
fgmask, cv2.MORPH_CLOSE, kernel, iterations=2)
|
||||
return cv2.bitwise_and(frame, frame, mask=fgmask)
|
||||
|
||||
def getMaskAreaRatio(self, mask):
|
||||
grayMask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY)
|
||||
ret, thresh = cv2.threshold(grayMask, 0, 255, 0)
|
||||
return np.sum(thresh)/(self.height*self.width*255)
|
||||
|
||||
def setupFrame(self, frame_width, frame_height):
|
||||
"""self.x0 and self.y0 are top left corner coordinates
|
||||
self.width and self.height are the width and height the ROI
|
||||
"""
|
||||
x, y = 0.0, 0.4
|
||||
self.x0 = int(frame_width*x)
|
||||
self.y0 = int(frame_height*y)
|
||||
self.width = 450
|
||||
self.height = 450
|
||||
|
||||
def countFingers(self, contour, contourAndHull):
|
||||
hull = cv2.convexHull(contour, returnPoints=False)
|
||||
if len(hull) > 3:
|
||||
defects = cv2.convexityDefects(contour, hull)
|
||||
cnt = 0
|
||||
if type(defects) != type(None):
|
||||
for i in range(defects.shape[0]):
|
||||
s, e, f, d = defects[i, 0]
|
||||
start = tuple(contour[s, 0])
|
||||
end = tuple(contour[e, 0])
|
||||
far = tuple(contour[f, 0])
|
||||
angle = self.calculateAngle(far, start, end)
|
||||
|
||||
|
||||
# Ignore the defects which are small and wide
|
||||
# Probably not fingers
|
||||
if d > 10000 and angle <= math.pi/2:
|
||||
cnt += 1
|
||||
cv2.circle(contourAndHull, far, 8, [255, 0, 0], -1)
|
||||
return True, cnt
|
||||
return False, 0
|
||||
|
||||
def calculateAngle(self, far, start, end):
|
||||
"""Cosine rule"""
|
||||
a = math.sqrt((end[0] - start[0])**2 + (end[1] - start[1])**2)
|
||||
b = math.sqrt((far[0] - start[0])**2 + (far[1] - start[1])**2)
|
||||
c = math.sqrt((end[0] - far[0])**2 + (end[1] - far[1])**2)
|
||||
angle = math.acos((b**2 + c**2 - a**2) / (2*b*c))
|
||||
return angle
|
||||
|
||||
def execute(self, cnt):
|
||||
if cnt == 1:
|
||||
self.mouse.click(pynput.mouse.Button.left, 1)
|
||||
if cnt == 2:
|
||||
self.mouse.click(pynput.mouse.Button.right, 1)
|
||||
if cnt == 3:
|
||||
pyautogui.press("down")
|
||||
elif cnt == 4:
|
||||
pyautogui.press("up")
|
||||
|
||||
def detectHand(self, frame, handHist):
|
||||
roi = frame[self.y0:self.y0 + self.height,
|
||||
self.x0:self.x0 + self.width,
|
||||
:]
|
||||
|
||||
roi = cv2.bilateralFilter(roi, 5, 50, 100)
|
||||
# Color masking
|
||||
histMask = self.histMasking(roi, handHist)
|
||||
cv2.imshow("histMask", histMask)
|
||||
|
||||
# Background substraction
|
||||
bgSubMask = self.bgSubMasking(roi)
|
||||
cv2.imshow("bgSubMask", bgSubMask)
|
||||
|
||||
# Attempt to learn the background automatically
|
||||
"""
|
||||
areaRatio = self.getMaskAreaRatio(bgSubMask)
|
||||
if areaRatio > 0.6:
|
||||
self.bgSubtractorLr = 1
|
||||
elif areaRatio < 0.001:
|
||||
self.bgSubtractorLr = 0
|
||||
"""
|
||||
|
||||
# Overall mask
|
||||
mask = cv2.bitwise_and(histMask, bgSubMask)
|
||||
|
||||
thresh = self.threshold(mask)
|
||||
cv2.imshow("Overall thresh", thresh)
|
||||
|
||||
contours, hierarchy = cv2.findContours(
|
||||
thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
|
||||
if len(contours) > 0:
|
||||
maxContour = self.getMaxContours(contours)
|
||||
M = cv2.moments(maxContour)
|
||||
if M["m00"] != 0:
|
||||
cX = int(M["m10"] / M["m00"])
|
||||
cY = int(M["m01"] / M["m00"])
|
||||
if self.dx != 0:
|
||||
self.mouse.move( -1 * 2 * (cX -self.dx), 2 * (cY - self.dy))
|
||||
self.dx = cX
|
||||
self.dy = cY
|
||||
# print(cX, cY)
|
||||
|
||||
# Draw contour and hull
|
||||
contourAndHull = np.zeros(roi.shape, np.uint8)
|
||||
hull = cv2.convexHull(maxContour)
|
||||
# print(type(hull))
|
||||
cv2.drawContours(contourAndHull, [maxContour], 0, (0, 255, 0), 2)
|
||||
cv2.drawContours(contourAndHull, [hull], 0, (0, 0, 255), 3)
|
||||
|
||||
found, cnt = self.countFingers(maxContour, contourAndHull)
|
||||
cv2.imshow("Contour and Hull", contourAndHull)
|
||||
|
||||
if found:
|
||||
self.execute(cnt)
|
||||
|
||||
centroid = self.getCentroid(maxContour)
|
||||
if centroid is not None:
|
||||
centroid[0] += self.x0
|
||||
centroid[1] += self.y0
|
||||
cv2.circle(frame, tuple(centroid), 5, [255, 0, 0], -1)
|
||||
|
||||
def startDetecting(self):
|
||||
cap = cv2.VideoCapture(0)
|
||||
|
||||
frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
|
||||
frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
||||
self.setupFrame(frame_width, frame_height)
|
||||
|
||||
while cap.isOpened():
|
||||
ret, frame = cap.read()
|
||||
|
||||
# Increase the contrast
|
||||
# frame = cv2.convertScaleAbs(frame, alpha=3, beta=-500)
|
||||
|
||||
# Gamma corection
|
||||
# Increase the contrast
|
||||
frame = cv2.LUT(frame, self.lookUpTable)
|
||||
|
||||
cv2.rectangle(frame, (self.x0, self.y0), (self.x0 +
|
||||
self.width - 1, self.y0 + self.height - 1), (255, 0, 0), 2)
|
||||
|
||||
k = cv2.waitKey(1) & 0xFF
|
||||
|
||||
if k == ord("z"):
|
||||
self.isHandHistCreated = True
|
||||
handHist = self.createHandHistogram(frame)
|
||||
elif k == ord('b'):
|
||||
self.bgSubtractor = cv2.createBackgroundSubtractorMOG2(
|
||||
30, self.bgSubThreshold)
|
||||
self.isBgCaptured = True
|
||||
elif k == ord("r"):
|
||||
self.bgSubtractor = None
|
||||
self.isBgCaptured = False
|
||||
|
||||
if self.isHandHistCreated and self.isBgCaptured:
|
||||
self.detectHand(frame, handHist)
|
||||
elif not self.isHandHistCreated:
|
||||
self.drawRect(frame)
|
||||
|
||||
cv2.imshow("Output", frame)
|
||||
if k == ord("q"):
|
||||
break
|
||||
elif k == ord("j"):
|
||||
self.y0 = min(self.y0 + 20, frame_height - self.height)
|
||||
elif k == ord("k"):
|
||||
self.y0 = max(self.y0 - 20, 0)
|
||||
elif k == ord("h"):
|
||||
self.x0 = max(self.x0 - 20, 0)
|
||||
elif k == ord("l"):
|
||||
self.x0 = min(self.x0 + 20, frame_width - self.width)
|
||||
|
||||
cap.release()
|
||||
cv2.destroyAllWindows()
|
||||
|
||||
|
||||
mouse = pynput.mouse.Controller()
|
||||
video = cv2.VideoCapture(0)
|
||||
frame = video.read()
|
||||
findFistInit = None
|
||||
centerX = 0
|
||||
centerY = 0
|
||||
|
||||
while True:
|
||||
cascadeHand = cv2.CascadeClassifier("hand.xml")
|
||||
cascadeFist = cv2.CascadeClassifier("fist.xml")
|
||||
|
||||
frame = video.read()[1]
|
||||
fmask = cv2.GaussianBlur(cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY), (21,21), 0)
|
||||
cv2.imshow("fmask", fmask)
|
||||
|
||||
|
||||
findFist = cascadeFist.detectMultiScale(fmask, scaleFactor = 1.1, minNeighbors = 75)
|
||||
|
||||
if findFistInit is None:
|
||||
findFistInit = findFist
|
||||
|
||||
if findFist == ():
|
||||
findHand = cascadeHand.detectMultiScale(fmask, scaleFactor = 1.1, minNeighbors = 10)
|
||||
|
||||
if findHand != ():
|
||||
mouse.click(pynput.mouse.Button.left, 1)
|
||||
|
||||
if findFist != () and findFistInit != ():
|
||||
|
||||
dx, dy, dw, dh = findFist[0][0] - findFistInit[0][0], findFist[0][1] - findFistInit[0][1], findFist[0][2] - findFistInit[0][2], findFist[0][3] - findFistInit[0][3]
|
||||
|
||||
centerX = (dx + dx + dw) / 2
|
||||
centerY = (dy + dy + dh) / 2
|
||||
mouse.move( -1 * 7 * dx, 7 * dy)
|
||||
|
||||
findFistInit = findFist
|
||||
|
||||
drawBox(findFist, frame, 0, 0, 255)
|
||||
drawBox(findHand, frame, 0, 255, 0)
|
||||
|
||||
frame = cv2.flip(frame, 1)
|
||||
key = openCamera(frame)
|
||||
|
||||
if(key == ord('q')):
|
||||
break
|
||||
|
||||
|
||||
exit(video)
|
||||
if __name__ == "__main__":
|
||||
detector = FingersNumberDetector()
|
||||
detector.startDetecting()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user