Creating an LBPH (Local Binary Pattern Histogram) face recognizer using FaceRecognizer class in JavaCV

In this article we will discuss how to create a Java based LBP face recognizer using JavaCV. JavaCV is a wrapper to the Open CV libraries for image processing. The workflow that I am trying to implement is as below

• The calling program will supply a bunch of images and a person name to this class ( learnNewFace(String personName, IplImage[] images)
• The class will store the images in a folder and map the person name to an integer id
• The class will then read all the images and person names and train the model
• Once training is complete client can invoke identifyFace method giving an image to find out the matching personName (identifyFace(IplImage image))

Step -1)
Install JDK and Eclipse ( I used JDK 7 on 64 bit Windows 7 PC)

Step-2)
Install OpenCV and JavaCV(details can be found at http://code.google.com/p/javacv/). One thing we need to keep in mind is that the bit-ness of JDK and JavaCV  and OpenCV must match. That means for a 64 bit JDK, you have to use the javacv-linux-x86_64 jar along with the other jar files. Also, please make sure you extract the opencv to C:\ drive. Extracting anywhere else is a nightmare, to setup the correct path variables.

Step -3)
Create a new Java project in Eclipse and add the following jar files to the class path
javacv.jar, javacpp.jar  and one of the javacv-linux-x86.jar,javacv-linux-x86_64.jar,javacv-macosx-x86_64.jar,javacv-windows-x86.jar,javacv-windows-x86_64.jar according to your operating system
The workspace should look like this

Step-4)
Create a new class LBPFaceRecognizer in a package and copy the below code
The code is given below
package com.test.face;

import static com.googlecode.javacv.cpp.opencv_contrib.createLBPHFaceRecognizer;
import static com.googlecode.javacv.cpp.opencv_core.CV_32SC1;
import static com.googlecode.javacv.cpp.opencv_core.IPL_DEPTH_8U;
import static com.googlecode.javacv.cpp.opencv_core.cvCreateImage;
import static com.googlecode.javacv.cpp.opencv_core.cvGetSize;
import static com.googlecode.javacv.cpp.opencv_core.cvLoad;
import static com.googlecode.javacv.cpp.opencv_core.cvSetImageROI;
import static com.googlecode.javacv.cpp.opencv_highgui.cvLoadImage;
import static com.googlecode.javacv.cpp.opencv_highgui.cvSaveImage;
import static com.googlecode.javacv.cpp.opencv_imgproc.CV_BGR2GRAY;
import static com.googlecode.javacv.cpp.opencv_imgproc.CV_INTER_LINEAR;
import static com.googlecode.javacv.cpp.opencv_imgproc.cvCvtColor;
import static com.googlecode.javacv.cpp.opencv_imgproc.cvEqualizeHist;
import static com.googlecode.javacv.cpp.opencv_imgproc.cvResize;
import static com.googlecode.javacv.cpp.opencv_objdetect.cvHaarDetectObjects;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.util.Iterator;
import java.util.Properties;
import java.util.Set;

import com.googlecode.javacpp.Loader;
import com.googlecode.javacv.cpp.opencv_contrib.FaceRecognizer;
import com.googlecode.javacv.cpp.opencv_contrib.FaceRecognizerPtr;
import com.googlecode.javacv.cpp.opencv_core.CvMat;
import com.googlecode.javacv.cpp.opencv_core.CvMemStorage;
import com.googlecode.javacv.cpp.opencv_core.CvRect;
import com.googlecode.javacv.cpp.opencv_core.CvSeq;
import com.googlecode.javacv.cpp.opencv_core.IplImage;
import com.googlecode.javacv.cpp.opencv_core.MatVector;
import com.googlecode.javacv.cpp.opencv_objdetect;
import com.googlecode.javacv.cpp.opencv_objdetect.CvHaarClassifierCascade;

public class LBPFaceRecognizer {



        private static String faceDataFolder = "C:\\test\\data\\";
        public static String imageDataFolder = faceDataFolder + "images\\";
        private static final String CASCADE_FILE = "C:\\opencv\\data\\haarcascades\\haarcascade_frontalface_alt.xml";
        private static final String frBinary_DataFile = faceDataFolder + "frBinary.dat";
        public static final String personNameMappingFileName = faceDataFolder + "personNumberMap.properties";

        final CvHaarClassifierCascade cascade = new CvHaarClassifierCascade(cvLoad(CASCADE_FILE));
        private Properties dataMap = new Properties();
        private static LBPFaceRecognizer instance = new LBPFaceRecognizer();

        public static final int NUM_IMAGES_PER_PERSON =10;
        double binaryTreshold = 100;
        int highConfidenceLevel = 70;

        FaceRecognizerPtr ptr_binary = null;
        private FaceRecognizer fr_binary = null;

        private LBPFaceRecognizer() {
                createModels();
                loadTrainingData();
        }

        public static LBPFaceRecognizer getInstance() {
                return instance;
        }

        private void createModels() {
                ptr_binary = createLBPHFaceRecognizer(1, 8, 8, 8, binaryTreshold);
                fr_binary = ptr_binary.get();
        }

        protected CvSeq detectFace(IplImage originalImage) {
                CvSeq faces = null;
                Loader.load(opencv_objdetect.class);
                try {
                        IplImage grayImage = IplImage.create(originalImage.width(), originalImage.height(), IPL_DEPTH_8U, 1);
                        cvCvtColor(originalImage, grayImage, CV_BGR2GRAY);
                        CvMemStorage storage = CvMemStorage.create();
                        faces = cvHaarDetectObjects(grayImage, cascade, storage, 1.1, 1, 0);

                } catch (Exception e) {
                        e.printStackTrace();
                }
                return faces;
        }

        public String identifyFace(IplImage image) {
                System.err.println("==========================================================");
                String personName = "";

                Set keys = dataMap.keySet();

                if (keys.size() > 0) {
                        int[] ids = new int[1];
                        double[] distance = new double[1];
                        int result = -1;

                                fr_binary.predict(image, ids, distance);
                                //just deriving a confidence number against treshold
                                result = ids[0];

                                if (result > -1 && distance[0]<highConfidenceLevel) {
                                        personName = (String) dataMap.get("" + result);
                                }
                }

                return personName;
        }


        //The logic to learn a new face is to store the recorded images to a folder and retrain the model
        //will be replaced once update feature is available
        public boolean learnNewFace(String personName, IplImage[] images) throws Exception {
                int memberCounter = dataMap.size();
                if(dataMap.containsValue(personName)){
                        Set keys = dataMap.keySet();
                        Iterator ite = keys.iterator();
                        while (ite.hasNext()) {
                                String personKeyForTraining = (String) ite.next();
                                String personNameForTraining = (String) dataMap.getProperty(personKeyForTraining);
                                if(personNameForTraining.equals(personName)){
                                        memberCounter = Integer.parseInt(personKeyForTraining);
                                        System.err.println("Person already exist.. re-learning..");
                                }
                        }
                }
                dataMap.put("" + memberCounter, personName);
                storeTrainingImages(personName, images);
                retrainAll();

                return true;
        }


        public IplImage preprocessImage(IplImage image, CvRect r){
                IplImage gray = cvCreateImage(cvGetSize(image), IPL_DEPTH_8U, 1);
                IplImage roi = cvCreateImage(cvGetSize(image), IPL_DEPTH_8U, 1);
                CvRect r1 = new CvRect(r.x()-10, r.y()-10, r.width()+10, r.height()+10);
                cvCvtColor(image, gray, CV_BGR2GRAY);
                cvSetImageROI(gray, r1);
                cvResize(gray, roi, CV_INTER_LINEAR);
                cvEqualizeHist(roi, roi);
                return roi;
        }

        private void retrainAll() throws Exception {
                Set keys = dataMap.keySet();
                if (keys.size() > 0) {
                        MatVector trainImages = new MatVector(keys.size() * NUM_IMAGES_PER_PERSON);
                        CvMat trainLabels = CvMat.create(keys.size() * NUM_IMAGES_PER_PERSON, 1, CV_32SC1);
                        Iterator ite = keys.iterator();
                        int count = 0;

                        System.err.print("Loading images for training...");
                        while (ite.hasNext()) {
                                String personKeyForTraining = (String) ite.next();
                                String personNameForTraining = (String) dataMap.getProperty(personKeyForTraining);
                                IplImage[] imagesForTraining = readImages(personNameForTraining);
                                IplImage grayImage = IplImage.create(imagesForTraining[0].width(), imagesForTraining[0].height(), IPL_DEPTH_8U, 1);

                                for (int i = 0; i < imagesForTraining.length; i++) {
                                        trainLabels.put(count, 0, Integer.parseInt(personKeyForTraining));
                                        cvCvtColor(imagesForTraining[i], grayImage, CV_BGR2GRAY);
                                        trainImages.put(count,grayImage);
                                        count++;
                                }
                                //storeNormalizedImages(personNameForTraining, imagesForTraining);
                        }

                        System.err.println("done.");

                        System.err.print("Training Binary model ....");
                        fr_binary.train(trainImages, trainLabels);
                        System.err.println("done.");
                        storeTrainingData();
                }

        }

        private void loadTrainingData() {

                try {
                        File personNameMapFile = new File(personNameMappingFileName);
                        if (personNameMapFile.exists()) {
                                FileInputStream fis = new FileInputStream(personNameMappingFileName);
                                dataMap.load(fis);
                                fis.close();
                        }

                        File binaryDataFile = new File(frBinary_DataFile);
                        System.err.print("Loading Binary model ....");
                        fr_binary.load(frBinary_DataFile);
                        System.err.println("done");


                } catch (Exception e) {
                        e.printStackTrace();
                }
        }

        private void storeTrainingData() throws Exception {
                System.err.print("Storing training models ....");

                File binaryDataFile = new File(frBinary_DataFile);
                if (binaryDataFile.exists()) {
                        binaryDataFile.delete();
                }
                fr_binary.save(frBinary_DataFile);

                File personNameMapFile = new File(personNameMappingFileName);
                if (personNameMapFile.exists()) {
                        personNameMapFile.delete();
                }
                FileOutputStream fos = new FileOutputStream(personNameMapFile, false);
                dataMap.store(fos, "");
                fos.close();

                System.err.println("done.");
        }


        public void storeTrainingImages(String personName, IplImage[] images) {
                for (int i = 0; i < images.length; i++) {
                        String imageFileName = imageDataFolder + "training\\" + personName + "_" + i + ".bmp";
                        File imgFile = new File(imageFileName);
                        if (imgFile.exists()) {
                                imgFile.delete();
                        }
                        cvSaveImage(imageFileName, images[i]);
                }
        }

        private IplImage[] readImages(String personName) {
                File imgFolder = new File(imageDataFolder);
                IplImage[] images = null;
                if (imgFolder.isDirectory() && imgFolder.exists()) {
                        images = new IplImage[NUM_IMAGES_PER_PERSON];
                        for (int i = 0; i < NUM_IMAGES_PER_PERSON; i++) {
                                String imageFileName = imageDataFolder + "training\\" + personName + "_" + i + ".bmp";
                                IplImage img = cvLoadImage(imageFileName);
                                images[i] = img;
                        }

                }
                return images;
        }


}

Step-5)
Write a client code. The client code must do the following

1)      Get an instance of the face recognizer
                        LBPFaceRecognizer fr = LBPFaceRecognizer.getInstance();
2)      Capture image and grab the face coordinates using the code described here
                        FrameGrabber grabber = new OpenCVFrameGrabber(0);
            grabber.start();
            IplImage img = grabber.grab();
            IplImage snapshot = cvCreateImage(cvGetSize(img), img.depth(), img.nChannels());
            cvFlip(img, snapshot, 1);
            CvSeq faces = fr.detectFace(img);
            CvRect r = new CvRect(cvGetSeqElem(faces, 0));

3)      Preprocess the image using fr.preprocess
                     trainImages[imageCounter] = fr.preprocessImage(img, r);
4)      Invoke the learn method
                     fr.learnNewFace("User1", trainImages);
5)      Once the learning is complete you can use fr.identiofyFace to identify a face
                    fr.identifyFace(fr.preprocessImage(img, r));

NOTE: The convenience method preprocessImage is used to clip face, resize and normalize image to improve accuracy.
Happy coding :)