diff --git a/.gitignore b/.gitignore index 7db61e6d..c6016a95 100644 --- a/.gitignore +++ b/.gitignore @@ -2,12 +2,13 @@ __pycache__/ *.py[cod] *$py.class - +**.idea/** # C extensions *.so # Distribution / packaging .Python +data/ build/ develop-eggs/ dist/ diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index e6b25fed..00000000 --- a/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM ubuntu:18.04 -RUN apt-get update -RUN apt install -y python3 -RUN apt install -y python3-pip -RUN apt install -y cmake -RUN apt install -y libsm6 -RUN apt install -y libxext6 -RUN apt install -y libxrender1 -RUN apt install -y libfontconfig1 -RUN pip3 install --upgrade pip -COPY . /home/GazeTracking -WORKDIR /home/GazeTracking -RUN pip3 install -r requirements.txt diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 37519b52..00000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2019 Antoine Lamé - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index 9377c5ce..64e5dcb7 100644 --- a/README.md +++ b/README.md @@ -1,172 +1,20 @@ -# Gaze Tracking +# Gaze Tracking for Guzy -![made-with-python](https://img.shields.io/badge/Made%20with-Python-1f425f.svg) -![Open Source Love](https://badges.frapsoft.com/os/v1/open-source.svg?v=103) -![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg) -[![GitHub stars](https://img.shields.io/github/stars/antoinelame/GazeTracking.svg?style=social)](https://github.com/antoinelame/GazeTracking/stargazers) +Based on: github.com/antoinelame/GazeTracking -This is a Python (2 and 3) library that provides a **webcam-based eye tracking system**. It gives you the exact position of the pupils and the gaze direction, in real time. +setting up: -[![Demo](https://i.imgur.com/WNqgQkO.gif)](https://youtu.be/YEZMk1P0-yw) +- create a virtual env with python==3.8 +e.g. using conda: ``conda create -n your_name python=3.8`` +- activate your virtual environment +- git clone this project +- install packages by running ``pip install -e .`` +- run the solution from command line with command: ``python video_analysis.py -p data/sample_input.mp4 -o sample_output.json`` -_🚀 Quick note: I'm looking for job opportunities as a software developer, for exciting projects in ambitious companies. Anywhere in the world. Send me an email!_ +please use ``python video_analysis.py -h`` for possible settings -## Installation +the output points are located under key 'points' in output json. -Clone this project: +*keep in mind that output needs to be a json.* -```shell -git clone https://github.com/antoinelame/GazeTracking.git -``` - -### For Pip install -Install these dependencies (NumPy, OpenCV, Dlib): - -```shell -pip install -r requirements.txt -``` - -> The Dlib library has four primary prerequisites: Boost, Boost.Python, CMake and X11/XQuartx. If you doesn't have them, you can [read this article](https://www.pyimagesearch.com/2017/03/27/how-to-install-dlib/) to know how to easily install them. - - -### For Anaconda install -Install these dependencies (NumPy, OpenCV, Dlib): - -```shell -conda env create --file environment.yml -#After creating environment, activate it -conda activate GazeTracking -``` - - -### Verify Installation - -Run the demo: - -```shell -python example.py -``` - -## Simple Demo - -```python -import cv2 -from gaze_tracking import GazeTracking - -gaze = GazeTracking() -webcam = cv2.VideoCapture(0) - -while True: - _, frame = webcam.read() - gaze.refresh(frame) - - new_frame = gaze.annotated_frame() - text = "" - - if gaze.is_right(): - text = "Looking right" - elif gaze.is_left(): - text = "Looking left" - elif gaze.is_center(): - text = "Looking center" - - cv2.putText(new_frame, text, (60, 60), cv2.FONT_HERSHEY_DUPLEX, 2, (255, 0, 0), 2) - cv2.imshow("Demo", new_frame) - - if cv2.waitKey(1) == 27: - break -``` - -## Documentation - -In the following examples, `gaze` refers to an instance of the `GazeTracking` class. - -### Refresh the frame - -```python -gaze.refresh(frame) -``` - -Pass the frame to analyze (numpy.ndarray). If you want to work with a video stream, you need to put this instruction in a loop, like the example above. - -### Position of the left pupil - -```python -gaze.pupil_left_coords() -``` - -Returns the coordinates (x,y) of the left pupil. - -### Position of the right pupil - -```python -gaze.pupil_right_coords() -``` - -Returns the coordinates (x,y) of the right pupil. - -### Looking to the left - -```python -gaze.is_left() -``` - -Returns `True` if the user is looking to the left. - -### Looking to the right - -```python -gaze.is_right() -``` - -Returns `True` if the user is looking to the right. - -### Looking at the center - -```python -gaze.is_center() -``` - -Returns `True` if the user is looking at the center. - -### Horizontal direction of the gaze - -```python -ratio = gaze.horizontal_ratio() -``` - -Returns a number between 0.0 and 1.0 that indicates the horizontal direction of the gaze. The extreme right is 0.0, the center is 0.5 and the extreme left is 1.0. - -### Vertical direction of the gaze - -```python -ratio = gaze.vertical_ratio() -``` - -Returns a number between 0.0 and 1.0 that indicates the vertical direction of the gaze. The extreme top is 0.0, the center is 0.5 and the extreme bottom is 1.0. - -### Blinking - -```python -gaze.is_blinking() -``` - -Returns `True` if the user's eyes are closed. - -### Webcam frame - -```python -frame = gaze.annotated_frame() -``` - -Returns the main frame with pupils highlighted. - -## You want to help? - -Your suggestions, bugs reports and pull requests are welcome and appreciated. You can also starring ⭐️ the project! - -If the detection of your pupils is not completely optimal, you can send me a video sample of you looking in different directions. I would use it to improve the algorithm. - -## Licensing - -This project is released by Antoine Lamé under the terms of the MIT Open Source License. View LICENSE for more information. +**keep in mind that the solution still needs calibration** \ No newline at end of file diff --git a/VERSION b/VERSION new file mode 100644 index 00000000..8a9ecc2e --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.0.1 \ No newline at end of file diff --git a/build_and_run.sh b/build_and_run.sh deleted file mode 100644 index 8e136e85..00000000 --- a/build_and_run.sh +++ /dev/null @@ -1,12 +0,0 @@ -IMAGE_NAME=gaze-tracking - -# allow root access to x server -xhost local:root -# build and run docker image -([ "$(docker images -q ${IMAGE_NAME})" == "" ] && docker build -t ${IMAGE_NAME} . ) -([ "$(docker images -q ${IMAGE_NAME})" != "" ] && \ -docker run --rm --device /dev/video0 \ - -e DISPLAY=${DISPLAY} \ - -v /tmp/.X11-unix:/tmp/.X11-unix \ - --env="QT_X11_NO_MITSHM=1" \ - -it ${IMAGE_NAME} bash) diff --git a/config.py b/config.py new file mode 100644 index 00000000..bdded358 --- /dev/null +++ b/config.py @@ -0,0 +1,9 @@ +EYE_MARGIN = 10 +# FOURTH_PERCENTILE_WEIGHT = 5 +# THIRD_PERCENTILE_WEIGHT = 1 +# PERCENTILE_WEIGHT = FOURTH_PERCENTILE_WEIGHT * THIRD_PERCENTILE_WEIGHT +SKEWNESS_WEIGHT = 10 +PUPIL_WEIGHT = 1 +ANGLE_TOTAL_WEIGHT = SKEWNESS_WEIGHT + PUPIL_WEIGHT +MAX_RATIO = 0.6 +MIN_RATIO = 0.4 diff --git a/environment.yml b/environment.yml deleted file mode 100644 index 1b9f4931..00000000 --- a/environment.yml +++ /dev/null @@ -1,9 +0,0 @@ -name: GazeTracking -channels: - - conda-forge - - anaconda - - defaults -dependencies: - - numpy == 1.16.1 - - opencv == 3.4.* - - dlib == 19.17.* diff --git a/example.py b/example.py deleted file mode 100644 index c91a67a3..00000000 --- a/example.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -Demonstration of the GazeTracking library. -Check the README.md for complete documentation. -""" - -import cv2 -from gaze_tracking import GazeTracking - -gaze = GazeTracking() -webcam = cv2.VideoCapture(0) - -while True: - # We get a new frame from the webcam - _, frame = webcam.read() - - # We send this frame to GazeTracking to analyze it - gaze.refresh(frame) - - frame = gaze.annotated_frame() - text = "" - - if gaze.is_blinking(): - text = "Blinking" - elif gaze.is_right(): - text = "Looking right" - elif gaze.is_left(): - text = "Looking left" - elif gaze.is_center(): - text = "Looking center" - - cv2.putText(frame, text, (90, 60), cv2.FONT_HERSHEY_DUPLEX, 1.6, (147, 58, 31), 2) - - left_pupil = gaze.pupil_left_coords() - right_pupil = gaze.pupil_right_coords() - cv2.putText(frame, "Left pupil: " + str(left_pupil), (90, 130), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) - cv2.putText(frame, "Right pupil: " + str(right_pupil), (90, 165), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) - - cv2.imshow("Demo", frame) - - if cv2.waitKey(1) == 27: - break - -webcam.release() -cv2.destroyAllWindows() diff --git a/gaze_tracking/eye.py b/gaze_tracking/eye.py index a3efdc06..6536a3c9 100644 --- a/gaze_tracking/eye.py +++ b/gaze_tracking/eye.py @@ -2,6 +2,7 @@ import numpy as np import cv2 from .pupil import Pupil +from config import * class Eye(object): @@ -54,7 +55,7 @@ def _isolate(self, frame, landmarks, points): eye = cv2.bitwise_not(black_frame, frame.copy(), mask=mask) # Cropping on the eye - margin = 5 + margin = EYE_MARGIN min_x = np.min(region[:, 0]) - margin max_x = np.max(region[:, 0]) + margin min_y = np.min(region[:, 1]) - margin @@ -117,3 +118,4 @@ def _analyze(self, original_frame, landmarks, side, calibration): threshold = calibration.threshold(side) self.pupil = Pupil(self.frame, threshold) + diff --git a/gaze_tracking/gaze_tracking.py b/gaze_tracking/gaze_tracking.py index d04769dd..6d66adf4 100644 --- a/gaze_tracking/gaze_tracking.py +++ b/gaze_tracking/gaze_tracking.py @@ -4,6 +4,9 @@ import dlib from .eye import Eye from .calibration import Calibration +import numpy as np +from config import * +from scipy.stats import skew class GazeTracking(object): @@ -82,9 +85,26 @@ def horizontal_ratio(self): the center is 0.5 and the extreme left is 1.0 """ if self.pupils_located: - pupil_left = self.eye_left.pupil.x / (self.eye_left.center[0] * 2 - 10) - pupil_right = self.eye_right.pupil.x / (self.eye_right.center[0] * 2 - 10) - return (pupil_left + pupil_right) / 2 + # get skew of eye. it helps determine whether usr is looking right or left + count_right = np.sum(self.eye_right.frame < 255, axis=0) + # fourth_25_per = count_right[-int(len(count_right) / 4):] + # third_25_per = count_right[int(len(count_right) / 2):-int(len(count_right)/4)] + # right_skewness = ((fourth_25_per.sum() * FOURTH_PERCENTILE_WEIGHT + + # third_25_per.sum() * THIRD_PERCENTILE_WEIGHT) + # / PERCENTILE_WEIGHT) / count_right.sum() + right_skewness = count_right[int(len(count_right) / 2):].sum() / count_right.sum() + count_left = np.sum(self.eye_left.frame < 255, axis=0) + # fourth_25_per = count_left[-int(len(count_left) / 4):] + # third_25_per = count_left[int(len(count_left) / 2):-int(len(count_left)/4)] + # left_skewness = ((fourth_25_per.sum() * FOURTH_PERCENTILE_WEIGHT + + # third_25_per.sum() * THIRD_PERCENTILE_WEIGHT) + # / PERCENTILE_WEIGHT) / count_left.sum() + left_skewness = count_left[int(len(count_left) / 2):].sum() / count_left.sum() + skewness = (right_skewness + left_skewness) / 2 + pupil_left = (self.eye_left.pupil.x - EYE_MARGIN) / (self.eye_left.center[0] * 2 - EYE_MARGIN) + pupil_right = (self.eye_right.pupil.x - EYE_MARGIN) / (self.eye_right.center[0] * 2 - EYE_MARGIN) + ratio = ((pupil_left + pupil_right) / 2 * PUPIL_WEIGHT + skewness * SKEWNESS_WEIGHT) / ANGLE_TOTAL_WEIGHT + return ratio def vertical_ratio(self): """Returns a number between 0.0 and 1.0 that indicates the diff --git a/requirements.txt b/requirements.txt index 657deaa6..47ab24d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ numpy == 1.22.0 -opencv_python == 4.2.0.32 +opencv_python == 4.8.1.78 dlib == 19.16.0 +matplotlib==3.7.3 +scipy==1.10.1 +tqdm==4.66.1 \ No newline at end of file diff --git a/sample_output.json b/sample_output.json new file mode 100644 index 00000000..fc6c0162 --- /dev/null +++ b/sample_output.json @@ -0,0 +1,59 @@ +{ + "timestamp": "2023-10-09 22:43:35", + "input_file": "data/sample_input.mp4", + "points": [ + 1277, + 1365, + 1423, + 1580, + 1552, + 1501, + 1678, + 1709, + 1671, + 1640, + 1643, + 1764, + 1656, + 1712, + 1710, + 1640, + 1710, + 1746, + 1667, + 1747, + 1719, + 1712, + 1736, + 1693, + 1664, + 1680, + 1705, + 1697, + 1654, + 1668, + 1442, + 1261, + 1189, + 1195, + 1252, + 1164, + 1138, + 1151, + 1137, + 1146, + 1164, + 1215, + 1153, + 1184, + 1183, + 1163, + 1132, + 1142, + 1206, + 1184, + 1096, + 1185, + 1151 + ] +} \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..b79dd154 --- /dev/null +++ b/setup.py @@ -0,0 +1,16 @@ +from setuptools import setup + +with open('requirements.txt', 'r') as f: + requirements = f.read().splitlines() + +with open('VERSION', 'r') as f: + VERSION = f.read().strip() + +setup( + name='GazeTrackingForGuzy', + version=VERSION, + author='Adam Mika', + description='Package for obtaining location on screen where user is looking.', + packages=['gaze_tracking'], + install_requires=requirements, # Specify requirements here +) diff --git a/video_analysis.py b/video_analysis.py new file mode 100644 index 00000000..54b69e5e --- /dev/null +++ b/video_analysis.py @@ -0,0 +1,153 @@ +import os +import cv2 +import matplotlib.pyplot as plt +from gaze_tracking import GazeTracking +from config import * +from tqdm import tqdm +import logging +import datetime +import argparse +import json + +log = logging.getLogger("video_analysis") +logging.basicConfig(level = logging.INFO) + +def calculate_circle_position(value, image_width): + # Define the left and right positions for the circle + left_position = MIN_RATIO + right_position = MAX_RATIO + + # Ensure the value is within the range [0.4, 0.6] + value = max(left_position, min(right_position, value)) + + # Calculate the horizontal position of the circle using linear interpolation + circle_x = int((value - left_position) / (right_position - left_position) * image_width) + + return circle_x + + +def analyze_video(path, max_x: int, output_path_video: str, show=False): + gaze = GazeTracking() + cap = cv2.VideoCapture(path) + total_frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + points = [] + if output_path_video: + frame_width = int(cap.get(3)) + frame_height = int(cap.get(4)) + size = (frame_width, frame_height) + result = cv2.VideoWriter(output_path_video, + cv2.VideoWriter_fourcc(*'MJPG'), + 10, size) + with tqdm(total=total_frame_count - 3) as pbar: + while cap.isOpened(): # if too slow on better computer, perhaps need to apply multithread + ret, frame = cap.read() + if ret: + gaze.refresh(frame) + ratio = gaze.horizontal_ratio() + if ratio: + circle_x = calculate_circle_position(ratio, max_x) + points.append(circle_x) + if show: + frame = cv2.circle(frame, (circle_x, int(frame.shape[0] / 2)), radius=20, color=(0, 0, 255), + thickness=-1) + if output_path_video: + result.write(frame) + if show: + cv2.imshow("Demo", frame) + if cv2.waitKey(25) & 0xFF == ord('q'): + break + pbar.update(1) + else: + break + cap.release() + if output_path_video: + result.release() + if show: + cv2.destroyAllWindows() + return points + + +def show_histogram(point_list: list, max_x: int) -> None: + n, bins, patches = plt.hist(point_list, density=False, bins=5) # density=False would make counts + plt.bar_label(patches) + plt.ylabel('Number of occurrence') + plt.xlim(xmax=max_x, xmin=0) + plt.xlabel('') + plt.show() + + +def str_to_bool(value): + if isinstance(value, bool): + return value + if value.lower() in ('true', 't', 'yes', 'y', '1'): + return True + elif value.lower() in ('false', 'f', 'no', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError("Invalid boolean value: {}".format(value)) + + +def main(): + # Create an ArgumentParser object + parser = argparse.ArgumentParser(description="Script for analyzing where a person is looking at the screen") + + # Add the -p or --path argument + parser.add_argument('-p', '--path', required=False, help="path to video clip", default='data/sample_input.mp4') + parser.add_argument('-o', '--output', required=False, default='data/output.json', + help="path to output json") + parser.add_argument('-s', '--show', required=False, type=str_to_bool, default=False, + help='Show the video during processing with dot drawn') + parser.add_argument('-x', '--max_x', required=False, default=1920, + help='Max horizontal resolution of the TV,default 1920') + parser.add_argument('--hist', required=False, type=str_to_bool, default=False, + help='Whether to show histogram after processing') + parser.add_argument('--video_output', required=False, default=None, + help='Path where video output (with dot printed should be saved') + + # Parse the command-line arguments + args = parser.parse_args() + + # Access the file path argument + file_path = args.path + output_path = args.output + show = args.show + max_x = args.max_x + show_hist = args.hist + output_path_video = args.video_output + + if not os.path.isfile(file_path): + raise FileNotFoundError(f'No file under location: {file_path}.Please check the path provided') + + if not output_path.lower().endswith(".json"): + raise TypeError('output file must be of json format') + output_directory = os.path.dirname(output_path) + if output_directory: + os.makedirs(output_directory, exist_ok=True) + if output_path_video: + output_video = os.path.dirname(output_path_video) + if output_video: + os.makedirs(output_video, exist_ok=True) + log.info('Starting Analysis...') + spotted_points = analyze_video(path=file_path, max_x=max_x, output_path_video=output_path_video, show=show) + log.info('Analysis completed!') + if show_hist: + show_histogram(spotted_points, max_x=max_x) + + output = { + 'timestamp': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + 'input_file': file_path, + 'points': spotted_points + } + + json_str = json.dumps(output, indent=4) + + # Write the JSON string to the output file + with open(output_path, "w") as output_file: + output_file.write(json_str) + + log.info(f'points spotted saved under {output_path} directory.') + return spotted_points + + +if __name__ == "__main__": + points = main()