Tuesday, October 30, 2012

MONODROID Camera preview as openGL texture

I've been pulling my hair out trying to get a simple camera preview on an android phone with an OpenGL sprite on top. This seems possible if you do it in eclipse using Java but my chosen platform, Monodroid and C# seems to have problems.

Later versions of the Android SDK seem to have this feature also but let's not forget that there are still a huge proportion of 2.3 phones out there that could be supported if only this was a simple feature.

Well, I brute-forced one that runs on OpenGL on a 2.3.3 phone.

#1 I created a camera preview listener and the associated classes to provide a nice .net event when the new camera frame arrived:


    public class CameraListener : Java.Lang.Object, Camera.IPreviewCallback
    {
        public event PreviewFrameHandler PreviewFrame;
        public void OnPreviewFrame(byte[] data, Camera camera)
        {
            if (PreviewFrame != null)
            {
                PreviewFrame(this, new PreviewFrameEventArgs(data, camera));
            }
        }
    }

    public delegate void PreviewFrameHandler(object sender, PreviewFrameEventArgs e);

    public class PreviewFrameEventArgs : EventArgs
    {
        byte[] _data;
        Camera _camera;

        public byte[] Data { get { return _data; } }

        public Camera Camera { get { return _camera; } }

        public PreviewFrameEventArgs(byte[] data, Camera camera)
        {
            _data = data;
            _camera = camera;
        }
    }

#2 Using MonoGame I created a basic app

    public class Game1 : Microsoft.Xna.Framework.Game
    {
        Camera _camera;
        CameraListener _listener = new CameraListener();

        GraphicsDeviceManager graphics;
        SpriteBatch spriteBatch;
        SpriteFont font;

        Texture2D _cameraBG;
        short[] _frameData = null;

        public Game1()
        {
            graphics = new GraphicsDeviceManager(this);

            Content.RootDirectory = "Content";

            graphics.IsFullScreen = true;
            graphics.PreferredBackBufferWidth = 800;
            graphics.PreferredBackBufferHeight = 480;
            graphics.SupportedOrientations = DisplayOrientation.LandscapeLeft | DisplayOrientation.LandscapeRight;

            _listener.PreviewFrame += new PreviewFrameHandler(_listener_PreviewFrame);
        }

        unsafe void _listener_PreviewFrame(object sender, PreviewFrameEventArgs e)
        {
            lock (this)
            {
                fixed (short* fd = &_frameData[0])
                {
                    fixed (byte* yuv = &e.Data[0])
                    {
                        YUV2RGB.convertYUV420_NV21toRGB5551(yuv, fd,
                                                            _cameraBG.Width, _cameraBG.Height);
                    }
                }
                _cameraBG.SetData(_frameData);
            }
        }

        /// <summary>
        /// Allows the game to perform any initialization it needs to before starting to run.
        /// This is where it can query for any required services and load any non-graphic
        /// related content.  Calling base.Initialize will enumerate through any components
        /// and initialize them as well.
        /// </summary>
        protected override void Initialize()
        {
            // TODO: Add your initialization logic here

            base.Initialize();

           
            _camera = Camera.Open();
            Camera.Parameters parameters = _camera.GetParameters();
            IList<Camera.Size> sizes = parameters.SupportedPreviewSizes;

            _cameraBG = new Texture2D(graphics.GraphicsDevice, sizes[9].Width, sizes[9].Height, false, SurfaceFormat.Bgra5551);
            _frameData=new short[sizes[9].Width*sizes[9].Height];
            parameters.SetPreviewSize(sizes[9].Width, sizes[9].Height);
            _camera.SetParameters(parameters);
           
            _camera.SetPreviewCallback(_listener);
            _camera.StartPreview();
        }

        /// <summary>
        /// LoadContent will be called once per game and is the place to load
        /// all of your content.
        /// </summary>
        protected override void LoadContent()
        {
            // Create a new SpriteBatch, which can be used to draw textures.
            spriteBatch = new SpriteBatch(GraphicsDevice);

            // TODO: use this.Content to load your game content here
            font = Content.Load<SpriteFont>("spriteFont1");
        }

        /// <summary>
        /// Allows the game to run logic such as updating the world,
        /// checking for collisions, gathering input, and playing audio.
        /// </summary>
        /// <param name="gameTime">Provides a snapshot of timing values.</param>
        protected override void Update(GameTime gameTime)
        {
            if (GamePad.GetState(PlayerIndex.One).Buttons.Back == ButtonState.Pressed)
            {
                Exit();
            }

            // TODO: Add your update logic here

            base.Update(gameTime);
        }

        int c = 0;

        /// <summary>
        /// This is called when the game should draw itself.
        /// </summary>
        /// <param name="gameTime">Provides a snapshot of timing values.</param>
        protected override void Draw(GameTime gameTime)
        {
            graphics.GraphicsDevice.Clear(Color.CornflowerBlue);


            spriteBatch.Begin();
            spriteBatch.Draw(_cameraBG, new Rectangle(0,0,graphics.GraphicsDevice.Viewport.Width, graphics.GraphicsDevice.Viewport.Height), Color.White);
            spriteBatch.DrawString(font, "Hello from MonoGame!", new Vector2(c++, 16), Color.White);
            spriteBatch.End();
            if (c > 350)
                c = 0;
            base.Draw(gameTime);
        }

#3 Using info gleaned from Wikipedia I put together a YUV to RGB translator NOTE the use of unsafe code. Sorry but that's the way it has to be. 

    public static class YUV2RGB
    {
        unsafe public static void convertYUV420_NV21toRGB4444(byte* yuvIn, short* rgbOut , int width, int height)
        {
            int size = width * height;
            int offset = size;
            int u, v, y1, y2, y3, y4;

            for (int i = 0, k = 0; i < size; i += 2, k += 2)
            {
                y1 = yuvIn[i];
                y2 = yuvIn[i + 1];
                y3 = yuvIn[width + i];
                y4 = yuvIn[width + i + 1];

                u = yuvIn[offset + k];
                v = yuvIn[offset + k + 1];
                u = u - 128;
                v = v - 128;

                convertYUVtoRGB4444(y1, u, v, rgbOut, i);
                convertYUVtoRGB4444(y2, u, v, rgbOut, (i + 1));
                convertYUVtoRGB4444(y3, u, v, rgbOut, (width + i));
                convertYUVtoRGB4444(y4, u, v, rgbOut, (width + i + 1));

                if (i != 0 && (i + 2) % width == 0)
                    i += width;
            }
        }

        unsafe private static void convertYUVtoRGB4444(int y, int u, int v, short* rgbOut, int index)
        {
            int r = y + (int)1.402f * v;
            int g = y - (int)(0.344f * u + 0.714f * v);
            int b = y + (int)1.772f * u;
            r = r > 255 ? 255 : r < 0 ? 0 : r;
            g = g > 255 ? 255 : g < 0 ? 0 : g;
            b = b > 255 ? 255 : b < 0 ? 0 : b;      

            rgbOut[index] = (short)(0xf000 | ((r&0xf0)<<8) | ((g&0x00f0)<<4) | b>>4);
        }

        unsafe public static void convertYUV420_NV21toRGB5551(byte* yuvIn, Int16* rgbOut, int width, int height)
        {
            int size = width * height;
            int offset = size;
            int u, v, y1, y2, y3, y4;

            for (int i = 0, k = 0; i < size; i += 2, k += 2)
            {
                y1 = yuvIn[i];
                y2 = yuvIn[i + 1];
                y3 = yuvIn[width + i];
                y4 = yuvIn[width + i + 1];

                u = yuvIn[offset + k];
                v = yuvIn[offset + k + 1];
                u = u - 128;
                v = v - 128;

                convertYUVtoRGB5551(y1, u, v, rgbOut, i);
                convertYUVtoRGB5551(y2, u, v, rgbOut, (i + 1));
                convertYUVtoRGB5551(y3, u, v, rgbOut, (width + i));
                convertYUVtoRGB5551(y4, u, v, rgbOut, (width + i + 1));

                if (i != 0 && (i + 2) % width == 0)
                    i += width;
            }
        }

        unsafe private static void convertYUVtoRGB5551(int y, int u, int v, Int16* rgbOut, int index)
        {
            int r = y + (int)1.402f * v;
            int g = y - (int)(0.344f * u + 0.714f * v);
            int b = y + (int)1.772f * u;
            r = r > 255 ? 255 : r < 0 ? 0 : r;
            g = g > 255 ? 255 : g < 0 ? 0 : g;
            b = b > 255 ? 255 : b < 0 ? 0 : b;      
            rgbOut[index]=(short)(((b&0xf8) << 8) |
                          ((g&0xf8) << 3) |
                          ((r >> 2) & 0x3e |
                          1)
                          );
        }
    }

------------------------------------------------------
There you go.. Check out section #2 where the preview frame event handler converts the data and copies it into the texture. This texture can then be used as a background texture on your game.

My lack of thanks goes out to the people on StackOverflow whose unhelpful responses to my pleas were the inspiration for "Do it yer frikkin self Bob"

Please feel free to enjoy the code in any way that pleases you.

Hey, if you find it useful +1 it please.



1 comment:

Unknown said...

I was having similar problems finding any resources or help with an augmented reality Monodroid app.

Finding out about the Monogame add-on / extension to the monodroid framework, together with this sample, is revealing a host of possibilities. Thank you for taking the time to post this for others!