I've been pulling my hair out trying to get a simple camera preview on an android phone with an OpenGL sprite on top. This seems possible if you do it in eclipse using Java but my chosen platform, Monodroid and C# seems to have problems.
Later versions of the Android SDK seem to have this feature also but let's not forget that there are still a huge proportion of 2.3 phones out there that could be supported if only this was a simple feature.
Well, I brute-forced one that runs on OpenGL on a 2.3.3 phone.
#1 I created a camera preview listener and the associated classes to provide a nice .net event when the new camera frame arrived:
public class CameraListener
: Java.Lang.Object, Camera.IPreviewCallback
{
public event PreviewFrameHandler
PreviewFrame;
public void OnPreviewFrame(byte[]
data, Camera camera)
{
if
(PreviewFrame != null)
{
PreviewFrame(this, new PreviewFrameEventArgs(data, camera));
}
}
}
public delegate void PreviewFrameHandler(object
sender, PreviewFrameEventArgs e);
public class PreviewFrameEventArgs
: EventArgs
{
byte[]
_data;
Camera
_camera;
public byte[] Data { get { return _data; } }
public Camera Camera { get
{ return _camera; } }
public PreviewFrameEventArgs(byte[] data, Camera
camera)
{
_data = data;
_camera = camera;
}
}
#2 Using MonoGame I created a basic app
public class Game1 :
Microsoft.Xna.Framework.Game
{
Camera
_camera;
CameraListener
_listener = new CameraListener();
GraphicsDeviceManager
graphics;
SpriteBatch
spriteBatch;
SpriteFont
font;
Texture2D
_cameraBG;
short[]
_frameData = null;
public
Game1()
{
graphics = new
GraphicsDeviceManager(this);
Content.RootDirectory = "Content";
graphics.IsFullScreen = true;
graphics.PreferredBackBufferWidth =
800;
graphics.PreferredBackBufferHeight
= 480;
graphics.SupportedOrientations = DisplayOrientation.LandscapeLeft | DisplayOrientation.LandscapeRight;
_listener.PreviewFrame += new PreviewFrameHandler(_listener_PreviewFrame);
}
unsafe void _listener_PreviewFrame(object
sender, PreviewFrameEventArgs e)
{
lock
(this)
{
fixed
(short* fd = &_frameData[0])
{
fixed
(byte* yuv = &e.Data[0])
{
YUV2RGB.convertYUV420_NV21toRGB5551(yuv, fd,
_cameraBG.Width, _cameraBG.Height);
}
}
_cameraBG.SetData(_frameData);
}
}
/// <summary>
/// Allows the game to perform any initialization it needs to
before starting to run.
/// This is where it can query for any required services and
load any non-graphic
/// related content.
Calling base.Initialize will enumerate through any components
/// and initialize them as well.
/// </summary>
protected
override void
Initialize()
{
// TODO: Add your initialization logic here
base.Initialize();
_camera = Camera.Open();
Camera.Parameters parameters = _camera.GetParameters();
IList<Camera.Size>
sizes = parameters.SupportedPreviewSizes;
_cameraBG = new Texture2D(graphics.GraphicsDevice,
sizes[9].Width, sizes[9].Height, false, SurfaceFormat.Bgra5551);
_frameData=new
short[sizes[9].Width*sizes[9].Height];
parameters.SetPreviewSize(sizes[9].Width, sizes[9].Height);
_camera.SetParameters(parameters);
_camera.SetPreviewCallback(_listener);
_camera.StartPreview();
}
/// <summary>
/// LoadContent will be called once per game and is the place
to load
/// all of your content.
/// </summary>
protected
override void
LoadContent()
{
// Create
a new SpriteBatch, which can be used to draw textures.
spriteBatch = new SpriteBatch(GraphicsDevice);
// TODO: use this.Content to load your game content here
font = Content.Load<SpriteFont>("spriteFont1");
}
/// <summary>
/// Allows the game to run logic such as updating the world,
/// checking for collisions, gathering input, and playing
audio.
/// </summary>
/// <param
name="gameTime">Provides a
snapshot of timing values.</param>
protected
override void
Update(GameTime gameTime)
{
if
(GamePad.GetState(PlayerIndex.One).Buttons.Back
== ButtonState.Pressed)
{
Exit();
}
// TODO: Add your update logic here
base.Update(gameTime);
}
int c =
0;
/// <summary>
/// This is called when the game should draw itself.
/// </summary>
/// <param
name="gameTime">Provides a
snapshot of timing values.</param>
protected
override void
Draw(GameTime gameTime)
{
graphics.GraphicsDevice.Clear(Color.CornflowerBlue);
spriteBatch.Begin();
spriteBatch.Draw(_cameraBG, new Rectangle(0,0,graphics.GraphicsDevice.Viewport.Width,
graphics.GraphicsDevice.Viewport.Height), Color.White);
spriteBatch.DrawString(font, "Hello from MonoGame!", new Vector2(c++,
16), Color.White);
spriteBatch.End();
if
(c > 350)
c = 0;
base.Draw(gameTime);
}
#3 Using info gleaned from Wikipedia I put together a YUV to RGB translator NOTE the use of unsafe code. Sorry but that's the way it has to be.
public static class YUV2RGB
{
unsafe public static void convertYUV420_NV21toRGB4444(byte* yuvIn, short*
rgbOut , int width, int
height)
{
int
size = width * height;
int
offset = size;
int
u, v, y1, y2, y3, y4;
for
(int i = 0, k = 0; i < size; i += 2, k += 2)
{
y1 = yuvIn[i];
y2 = yuvIn[i + 1];
y3 = yuvIn[width + i];
y4 = yuvIn[width + i + 1];
u = yuvIn[offset + k];
v =
yuvIn[offset + k + 1];
u = u - 128;
v = v - 128;
convertYUVtoRGB4444(y1, u, v, rgbOut, i);
convertYUVtoRGB4444(y2, u, v, rgbOut, (i + 1));
convertYUVtoRGB4444(y3, u, v, rgbOut, (width + i));
convertYUVtoRGB4444(y4, u, v, rgbOut, (width + i + 1));
if (i != 0 && (i + 2) %
width == 0)
i += width;
}
}
unsafe private static void convertYUVtoRGB4444(int
y, int u, int
v, short* rgbOut, int
index)
{
int r = y + (int)1.402f * v;
int g = y - (int)(0.344f * u + 0.714f * v);
int b = y + (int)1.772f * u;
r = r > 255 ? 255 : r < 0 ? 0 : r;
g = g > 255 ? 255 : g < 0 ? 0 : g;
b = b > 255 ? 255 : b < 0 ? 0 : b;
rgbOut[index] = (short)(0xf000 | ((r&0xf0)<<8) |
((g&0x00f0)<<4) | b>>4);
}
unsafe public static void convertYUV420_NV21toRGB5551(byte* yuvIn, Int16*
rgbOut, int width, int
height)
{
int
size = width * height;
int
offset = size;
int
u, v, y1, y2, y3, y4;
for
(int i = 0, k = 0; i < size; i += 2, k += 2)
{
y1 = yuvIn[i];
y2 = yuvIn[i + 1];
y3 = yuvIn[width + i];
y4 = yuvIn[width + i + 1];
u = yuvIn[offset + k];
v = yuvIn[offset + k + 1];
u = u - 128;
v = v - 128;
convertYUVtoRGB5551(y1, u, v, rgbOut, i);
convertYUVtoRGB5551(y2, u, v, rgbOut, (i + 1));
convertYUVtoRGB5551(y3, u, v, rgbOut, (width + i));
convertYUVtoRGB5551(y4, u, v, rgbOut, (width + i + 1));
if (i != 0 && (i + 2) %
width == 0)
i += width;
}
}
unsafe private static void convertYUVtoRGB5551(int
y, int u, int
v, Int16* rgbOut, int
index)
{
int
r = y + (int)1.402f * v;
int
g = y - (int)(0.344f * u + 0.714f * v);
int
b = y + (int)1.772f * u;
r = r > 255 ? 255 : r < 0 ? 0
: r;
g = g > 255 ? 255 : g < 0 ? 0
: g;
b = b > 255 ? 255 : b < 0 ? 0
: b;
rgbOut[index]=(short)(((b&0xf8) << 8) |
((g&0xf8)
<< 3) |
((r >> 2) &
0x3e |
1)
);
}
}
------------------------------------------------------
There you go.. Check out section #2 where the preview frame event handler converts the data and copies it into the texture. This texture can then be used as a background texture on your game.
My lack of thanks goes out to the people on StackOverflow whose unhelpful responses to my pleas were the inspiration for "Do it yer frikkin self Bob"
Please feel free to enjoy the code in any way that pleases you.
Hey, if you find it useful +1 it please.
