This is a very simple head-tracking using OpenCV and only Viola Jones’ face detection framework.

Basically the idea is to use face / head tracking to create an immersive 3D experience. This, is of course is still a very early prototype.

I whipped this up in less than an hour, so it’s very dirty and lack of any optimization. You can make the whole thing faster using CamShift algorithm, and some cleanup on the OpenGL codes.

Have fun !

// // main.cpp // SimpleImmersion // // Created by Saburo Okita on 13/1/13. // Copyright (c) 2013 Saburo Okita. All Rights Reserved // #include <iostream> #include <string> #include <opencv2/opencv.hpp> #include "opencv2/objdetect/objdetect.hpp" #include "opencv2/highgui/highgui.hpp" #include "opencv2/imgproc/imgproc.hpp" #if (XN_PLATFORM == XN_PLATFORM_MACOSX) #include <GLUT/glut.h> #else #include <GL/glut.h> #endif using namespace std; using namespace cv; double viewDistance = 8.0; double viewAngle = 0.0; double prevMouseX = -1; double prevMouseY = -1; double mouseDeltaX = 0.0; double mouseDeltaY = 0.0; double angle = 0.0; Rect found(0, 0, 0, 0); void initGL() ; void reshape( int width, int height ); void display(); void idle(); void keyboardEvent( unsigned char key, int x, int y ); vector<Rect> faces; CvCapture * capture; Mat frame; CascadeClassifier face_cascade; string window_name = "Capture - Face detection"; int main(int argc, char ** argv ) { string face_cascade_name = "/opt/local/share/OpenCV/haarcascades/haarcascade_frontalface_default.xml"; face_cascade.load( face_cascade_name ); capture = cvCaptureFromCAM( -1 ); glutInit( &argc, argv ); glutInitDisplayMode( GLUT_DOUBLE | GLUT_RGBA ); glutInitWindowSize( 800, 600 ); glutCreateWindow( "" ); initGL(); glutKeyboardFunc( keyboardEvent ); glutDisplayFunc( display ); glutReshapeFunc( reshape ); glutIdleFunc( idle ); glutMainLoop(); return 0; } void display() { glClear( GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT ); glColor3f( 0.0f, 1.0f, 0.0f ); glMatrixMode(GL_MODELVIEW); glLoadIdentity(); double radians = (M_PI * viewAngle) / 180.0; gluLookAt( viewDistance * sin(radians), 0.0, viewDistance * cos(radians), 0.0, 0.0, 0.0, 0.0, 1.0, 0.0 ); glLineWidth( 2.0f ); glColor3f( 1.0f, 1.0f, 1.0f ); glBegin( GL_LINES ); for( int i = -20; i < 21; i++ ) { glVertex3f( i * 1.0f, -2.0f, -10.0f ); glVertex3f( i * 1.0f, -2.0f, 10.0f ); glVertex3f( i * 1.0f, -2.0f, -10.0f ); glVertex3f( i * 1.0f, 10.0f, -10.0f ); } glEnd(); //angle += 0.1; //if( angle >= 360.0 ) // angle = 0.0; glPushMatrix(); //glRotated( angle, 0.0, 1.0, 0.0 ); glBegin( GL_QUADS ); glNormal3f( 0.0, 0.0, 1.0f ); glVertex3f( -1.0f, -1.0f, 1.0f ); glVertex3f( 1.0f, -1.0f, 1.0f ); glVertex3f( 1.0f, 1.0f, 1.0f ); glVertex3f( -1.0f, 1.0f, 1.0f ); glNormal3f( -1.0, 0.0, 0.0f ); glVertex3f( -1.0f, -1.0f, 1.0f ); glVertex3f( -1.0f, 1.0f, 1.0f ); glVertex3f( -1.0f, 1.0f, -1.0f ); glVertex3f( -1.0f, -1.0f, -1.0f ); glNormal3f( 1.0, 0.0, 0.0f ); glVertex3f( 1.0f, -1.0f, 1.0f ); glVertex3f( 1.0f, -1.0f, -1.0f ); glVertex3f( 1.0f, 1.0f, -1.0f ); glVertex3f( 1.0f, 1.0f, 1.0f ); glNormal3f( 0.0, 0.0, -1.0f ); glVertex3f( -1.0f, -1.0f, -1.0f ); glVertex3f( -1.0f, 1.0f, -1.0f ); glVertex3f( 1.0f, 1.0f, -1.0f ); glVertex3f( 1.0f, -1.0f, -1.0f ); glEnd(); glPopMatrix(); glutSwapBuffers(); } void keyboardEvent( unsigned char key, int x, int y ) { if( key == 27 ) exit( 1 ); } void reshape( int width, int height ) { glMatrixMode( GL_PROJECTION ); glLoadIdentity(); glViewport(0, 0, width, height); gluPerspective( 45.0, (GLfloat) width / (GLfloat) height, 0.1f, 100.0f ); glMatrixMode( GL_MODELVIEW ); glLoadIdentity(); } void initGL() { glShadeModel( GL_SMOOTH ); glClearColor( 0.0f, 0.0f, 0.0f, 1.0f ); glClearDepth( 1.0f ); glEnable( GL_DEPTH_TEST ); glEnable( GL_TEXTURE_2D ); glDepthFunc( GL_LEQUAL ); glHint( GL_PERSPECTIVE_CORRECTION_HINT, GL_NICEST ); glHint( GL_POLYGON_SMOOTH_HINT, GL_NICEST ); glEnable( GL_CULL_FACE ); glEnable( GL_LIGHTING ); glEnable( GL_LIGHT0 ); GLfloat LightAmbient[]= { 1.0f, 1.0f, 1.0f, 1.0f }; GLfloat LightDiffuse[]= { 1.0f, 1.0f, 1.0f, 1.0f }; GLfloat LightPosition[]= { 5.0f, 5.0f, -1.0f, 1.0f }; glLightfv(GL_LIGHT0, GL_AMBIENT, LightAmbient); glLightfv(GL_LIGHT0, GL_DIFFUSE, LightDiffuse); glLightfv(GL_LIGHT0, GL_POSITION,LightPosition); } void idle() { glutPostRedisplay(); frame = cvQueryFrame( capture ); if( !frame.empty() ) { flip(frame, frame, 1); frame = frame.rowRange(150, 440); faces.clear(); Mat gray; cvtColor( frame, gray, CV_BGR2GRAY ); equalizeHist( gray, gray ); face_cascade.detectMultiScale( gray, faces, 1.1, 2, 0|CV_HAAR_SCALE_IMAGE, Size(30, 30) ); found.width = 0; for( int i = 0; i < faces.size(); i++ ) { if( faces[i].width > 200 ) { found = faces[i]; break; } } if( found.width > 200 ) { int delta_x = 200 - (found.x + found.width) / 2; float new_view_angle = -delta_x / 2.0; if( abs(viewAngle - new_view_angle) > 2.0 ) { viewAngle = new_view_angle; if( viewAngle > 360.0 ) viewAngle -= 360.0; if( viewAngle < 0.0 ) viewAngle += 360.0; } rectangle( frame, found, Scalar(255, 0, 0) ); } resize(frame, frame, Size( frame.cols * 0.5, frame.rows * 0.5 )); imshow( window_name, frame ); } }