From 25d63d6c822a5aa3483e22c0ebb16f889276b666 Mon Sep 17 00:00:00 2001 From: ryancheu Date: Mon, 13 May 2013 01:47:29 -0400 Subject: [PATCH 1/3] Modified live streaming example to include audio. Files a little messy, need some cleaning up --- .../video/stream/agent/StreamClient.java | 69 +++++- .../video/stream/agent/StreamServerAgent.java | 84 ++++++- .../stream/handler/H264StreamDecoder.java | 205 +++++++++++------- .../stream/handler/H264StreamEncoder.java | 94 +++++++- .../stream/handler/StreamFrameListener.java | 4 + 5 files changed, 375 insertions(+), 81 deletions(-) diff --git a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamClient.java b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamClient.java index 41f32e7c..286f3270 100644 --- a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamClient.java +++ b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamClient.java @@ -4,9 +4,17 @@ import java.awt.image.BufferedImage; import java.net.InetSocketAddress; +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.DataLine; +import javax.sound.sampled.LineUnavailableException; +import javax.sound.sampled.SourceDataLine; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.xuggle.xuggler.IAudioSamples; + import us.sosia.video.stream.agent.ui.SingleVideoDisplayWindow; import us.sosia.video.stream.handler.StreamFrameListener; @@ -14,6 +22,8 @@ public class StreamClient { /** * @author kerr * */ + private static SourceDataLine mLine; + private static boolean isFirst = true; private final static Dimension dimension = new Dimension(320,240); private final static SingleVideoDisplayWindow displayWindow = new SingleVideoDisplayWindow("Stream example",dimension); protected final static Logger logger = LoggerFactory.getLogger(StreamClient.class); @@ -25,6 +35,12 @@ public static void main(String[] args) { logger.info("setup dimension :{}",dimension); StreamClientAgent clientAgent = new StreamClientAgent(new StreamFrameListenerIMPL(),dimension); clientAgent.connect(new InetSocketAddress("localhost", 20000)); + try { + openJavaSound(); + } catch (LineUnavailableException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } @@ -35,8 +51,59 @@ public void onFrameReceived(BufferedImage image) { logger.info("frame received :{}",count++); displayWindow.updateImage(image); } - + @Override + public void onAudioRecieved(IAudioSamples samples) { + playJavaSound(samples); + // TODO Auto-generated method stub + + } } + private static void openJavaSound() throws LineUnavailableException + { + AudioFormat audioFormat = new AudioFormat( + AudioFormat.Encoding.PCM_SIGNED, + 44100.0F, 16, 2, 4, 44100, false); + + AudioFormat format = new AudioFormat(44100, + 16, + 2, + true, /* xuggler defaults to signed 16 bit samples */ + false); + DataLine.Info info = new DataLine.Info(SourceDataLine.class, format); + mLine = (SourceDataLine) AudioSystem.getLine(info); + /** + * if that succeeded, try opening the line. + */ + mLine.open(audioFormat); + /** + * And if that succeed, start the line. + */ + mLine.start(); + + + } + + private static synchronized void playJavaSound(IAudioSamples aSamples) + { + if (isFirst) { + isFirst = false; + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + /** + * We're just going to dump all the samples into the line. + */ + byte[] rawBytes = aSamples.getData().getByteArray(0, aSamples.getSize()); + System.out.println("got: " + rawBytes.length ); + mLine.write(rawBytes, 0, rawBytes.length); + + } + } diff --git a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamServerAgent.java b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamServerAgent.java index ba925bf6..84a331ba 100644 --- a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamServerAgent.java +++ b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamServerAgent.java @@ -2,7 +2,10 @@ import java.awt.Dimension; import java.awt.image.BufferedImage; +import java.io.ByteArrayOutputStream; import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -10,6 +13,12 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.DataLine; +import javax.sound.sampled.LineUnavailableException; +import javax.sound.sampled.TargetDataLine; + import org.jboss.netty.bootstrap.ServerBootstrap; import org.jboss.netty.channel.Channel; import org.jboss.netty.channel.group.ChannelGroup; @@ -28,15 +37,17 @@ public class StreamServerAgent implements IStreamServerAgent{ protected final static Logger logger = LoggerFactory.getLogger(StreamServer.class); protected final Webcam webcam; protected final Dimension dimension; - protected final ChannelGroup channelGroup = new DefaultChannelGroup(); + protected final static ChannelGroup channelGroup = new DefaultChannelGroup(); protected final ServerBootstrap serverBootstrap; //I just move the stream encoder out of the channel pipeline for the performance protected final H264StreamEncoder h264StreamEncoder; + protected final H264StreamEncoder secondEncoder; protected volatile boolean isStreaming; protected ScheduledExecutorService timeWorker; protected ExecutorService encodeWorker; protected int FPS = 25; protected ScheduledFuture imageGrabTaskFuture; + protected TargetDataLine line; public StreamServerAgent(Webcam webcam, Dimension dimension) { super(); this.webcam = webcam; @@ -52,6 +63,29 @@ public StreamServerAgent(Webcam webcam, Dimension dimension) { this.timeWorker = new ScheduledThreadPoolExecutor(1); this.encodeWorker = Executors.newSingleThreadExecutor(); this.h264StreamEncoder = new H264StreamEncoder(dimension, false); + this.secondEncoder = new H264StreamEncoder(dimension, false); + + AudioFormat format = new AudioFormat( + AudioFormat.Encoding.PCM_SIGNED, + 44100.0F, 16, 2, 4, 44100, false); + + this.line = null; + DataLine.Info info = new DataLine.Info(TargetDataLine.class, + + format); // format is an AudioFormat object + if (!AudioSystem.isLineSupported(info)) { + // Handle the error ... + } + // Obtain and open the line. + try { + + this.line = (TargetDataLine) AudioSystem.getLine(info); + this.line.open(format); + } catch (LineUnavailableException ex) { + // Handle the error ... + } + + this.line.start(); } @@ -80,6 +114,18 @@ public void stop() { serverBootstrap.releaseExternalResources(); } + private static void writeData( Object data) { + channelGroup.write(data); + /* + try { + Thread.sleep(10); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + */ + } + private class StreamServerListenerIMPL implements StreamServerListener{ @@ -97,6 +143,8 @@ public void onClientConnectedIn(Channel channel) { 1000/FPS, TimeUnit.MILLISECONDS); imageGrabTaskFuture = imageGrabFuture; + AudioGrabTask at = new AudioGrabTask(); + at.start(); isStreaming = true; } logger.info("current connected clients :{}",channelGroup.size()); @@ -151,6 +199,35 @@ public void run() { } + public class AudioGrabTask extends Thread { + public void run() { + int numBytesRead; + byte[] data = new byte [4096]; + while (true) { + if (data.length > 0 ){ + System.out.println("start read: " + System.currentTimeMillis()); + numBytesRead = line.read(data, 0, 4096); + System.out.println("end read: " + System.currentTimeMillis()); + Object msg2 = null; + try { + if ( numBytesRead > 0 ) { + msg2 = secondEncoder.encode(data,numBytesRead); + } + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + if ( msg2 != null ) { + System.out.println("writing audio"); + writeData(msg2); + } + Thread.yield(); + } + } + } + } + + private class EncodeTask implements Runnable{ private final BufferedImage image; @@ -161,11 +238,12 @@ public EncodeTask(BufferedImage image) { @Override public void run() { - try { + try { Object msg = h264StreamEncoder.encode(image); if (msg != null) { - channelGroup.write(msg); + writeData(msg); } + } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); diff --git a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamDecoder.java b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamDecoder.java index 4c4a9d0f..5be1a8b1 100644 --- a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamDecoder.java +++ b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamDecoder.java @@ -15,6 +15,7 @@ import us.sosia.video.stream.handler.frame.FrameDecoder; import com.xuggle.ferry.IBuffer; +import com.xuggle.xuggler.IAudioSamples; import com.xuggle.xuggler.ICodec; import com.xuggle.xuggler.IPacket; import com.xuggle.xuggler.IPixelFormat; @@ -30,6 +31,7 @@ public class H264StreamDecoder extends OneToOneDecoder{ protected final static Logger logger = LoggerFactory.getLogger(H264StreamDecoder.class); protected final IStreamCoder iStreamCoder = IStreamCoder.make(Direction.DECODING, ICodec.ID.CODEC_ID_H264); + protected final IStreamCoder iAudioStreamCoder = IStreamCoder.make(Direction.DECODING, ICodec.ID.CODEC_ID_AAC); protected final Type type = ConverterFactory.findRegisteredConverter(ConverterFactory.XUGGLER_BGR_24); protected final StreamFrameListener streamFrameListener; protected final Dimension dimension; @@ -83,6 +85,7 @@ private void initialize(){ //iStreamCoder.setAutomaticallyStampPacketsForStream(true); //iStreamCoder.setTimeBase(IRational.make(rate.getDenominator(),rate.getNumerator())); iStreamCoder.open(null, null); + iAudioStreamCoder.open(null, null); } @@ -111,8 +114,11 @@ protected Object decode(ChannelHandlerContext ctx, Channel channel, frameBuffer = (ChannelBuffer)msg; } + + int size = frameBuffer.readableBytes(); logger.info("decode the frame size :{}",size); + //start to decode IBuffer iBuffer = IBuffer.make(null, size); IPacket iPacket = IPacket.make(iBuffer); @@ -121,43 +127,87 @@ protected Object decode(ChannelHandlerContext ctx, Channel channel, if (!iPacket.isComplete()) { return null; } + logger.info("packet stream index: " + iPacket.getFlags()); - IVideoPicture picture = IVideoPicture.make(IPixelFormat.Type.YUV420P, - dimension.width, dimension.height); - try { - // decode the packet into the video picture - int postion = 0; - int packageSize = iPacket.getSize(); - while(postion < packageSize){ - postion+= iStreamCoder.decodeVideo(picture, iPacket, postion); - if (postion < 0) - throw new RuntimeException("error " - + " decoding video"); - // if this is a complete picture, dispatch the picture - if (picture.isComplete()){ - IConverter converter = ConverterFactory.createConverter(type - .getDescriptor(), picture); - BufferedImage image = converter.toImage(picture); - //BufferedImage convertedImage = ImageUtils.convertToType(image, BufferedImage.TYPE_3BYTE_BGR); - //here ,put out the image - if (streamFrameListener != null) { - streamFrameListener.onFrameReceived(image); - } - converter.delete(); - }else{ - picture.delete(); - iPacket.delete(); - return null; - } - //clean the picture and reuse it - picture.getByteBuffer().clear(); - } - } finally { - if (picture != null) - picture.delete(); - iPacket.delete(); - // ByteBufferUtil.destroy(data); - } + if ( iPacket.getByteBuffer().get() != -1 ) { + + IVideoPicture picture = IVideoPicture.make(IPixelFormat.Type.YUV420P, + dimension.width, dimension.height); + try { + // decode the packet into the video picture + int postion = 0; + int packageSize = iPacket.getSize(); + while(postion < packageSize){ + postion+= iStreamCoder.decodeVideo(picture, iPacket, postion); + if (postion < 0) + throw new RuntimeException("error " + + " decoding video"); + // if this is a complete picture, dispatch the picture + if (picture.isComplete()){ + IConverter converter = ConverterFactory.createConverter(type + .getDescriptor(), picture); + BufferedImage image = converter.toImage(picture); + //BufferedImage convertedImage = ImageUtils.convertToType(image, BufferedImage.TYPE_3BYTE_BGR); + //here ,put out the image + if (streamFrameListener != null) { + streamFrameListener.onFrameReceived(image); + } + converter.delete(); + }else{ + picture.delete(); + iPacket.delete(); + return null; + } + //clean the picture and reuse it + picture.getByteBuffer().clear(); + } + } finally { + if (picture != null) + picture.delete(); + iPacket.delete(); + // ByteBufferUtil.destroy(data); + } + } + else { + //System.out.println("decoding audio"); + IAudioSamples samples = IAudioSamples.make(1024, 1); + + /* + * A packet can actually contain multiple sets of samples (or frames of samples + * in audio-decoding speak). So, we may need to call decode audio multiple + * times at different offsets in the packet's data. We capture that here. + */ + int offset = 0; + + /* + * Keep going until we've processed all data + */ + while(offset < iPacket.getSize()) + { + //System.out.println("starting decode"); + int bytesDecoded = iAudioStreamCoder.decodeAudio(samples, iPacket, offset); + if (bytesDecoded < 0) + throw new RuntimeException("got error decoding audio in stream"); + + System.out.println("decoded :" + bytesDecoded); + offset += bytesDecoded; + + //System.out.println("decoded some bytes"); + /* + * Some decoder will consume data in a packet, but will not be able to construct + * a full set of samples yet. Therefore you should always check if you + * got a complete set of samples from the decoder + */ + if (samples.isComplete()) + { + if (streamFrameListener != null) { + streamFrameListener.onAudioRecieved(samples); + } + } + } + + System.out.println("got audio frame"); + } return null; } } @@ -202,45 +252,50 @@ public void run() { if (!iPacket.isComplete()) { return ; } - - - IVideoPicture picture = IVideoPicture.make(IPixelFormat.Type.YUV420P, - dimension.width, dimension.height); - try { - // decode the packet into the video picture - int postion = 0; - int packageSize = iPacket.getSize(); - while(postion < packageSize){ - postion+= iStreamCoder.decodeVideo(picture, iPacket, postion); - if (postion < 0) - throw new RuntimeException("error " - + " decoding video"); - // if this is a complete picture, dispatch the picture - if (picture.isComplete()){ - IConverter converter = ConverterFactory.createConverter(type - .getDescriptor(), picture); - BufferedImage image = converter.toImage(picture); - //BufferedImage convertedImage = ImageUtils.convertToType(image, BufferedImage.TYPE_3BYTE_BGR); - //here ,put out the image - if (streamFrameListener != null) { - streamFrameListener.onFrameReceived(image); - } - converter.delete(); - }else{ - picture.delete(); - iPacket.delete(); - return ; - } - //clean the picture and reuse it - picture.getByteBuffer().clear(); - } - } finally { - if (picture != null) - picture.delete(); - iPacket.delete(); - // ByteBufferUtil.destroy(data); - } - return ; + logger.info("packet stream index: " + iPacket.getStreamIndex()); + if ( iPacket.getStreamIndex() == 0 ) { + IVideoPicture picture = IVideoPicture.make(IPixelFormat.Type.YUV420P, + dimension.width, dimension.height); + try { + // decode the packet into the video picture + int postion = 0; + int packageSize = iPacket.getSize(); + while(postion < packageSize){ + postion+= iStreamCoder.decodeVideo(picture, iPacket, postion); + if (postion < 0) + throw new RuntimeException("error " + + " decoding video"); + // if this is a complete picture, dispatch the picture + if (picture.isComplete()){ + IConverter converter = ConverterFactory.createConverter(type + .getDescriptor(), picture); + BufferedImage image = converter.toImage(picture); + //BufferedImage convertedImage = ImageUtils.convertToType(image, BufferedImage.TYPE_3BYTE_BGR); + //here ,put out the image + if (streamFrameListener != null) { + streamFrameListener.onFrameReceived(image); + } + converter.delete(); + }else{ + picture.delete(); + iPacket.delete(); + return ; + } + //clean the picture and reuse it + picture.getByteBuffer().clear(); + } + } finally { + if (picture != null) + picture.delete(); + iPacket.delete(); + // ByteBufferUtil.destroy(data); + } + return ; + } + else { + iPacket.delete(); + logger.info("got audio frame"); + } } } diff --git a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamEncoder.java b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamEncoder.java index f4448e99..b5bd3f15 100644 --- a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamEncoder.java +++ b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamEncoder.java @@ -4,6 +4,9 @@ import java.awt.image.BufferedImage; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.Arrays; + +import javax.sound.sampled.AudioFormat; import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.buffer.ChannelBuffers; @@ -15,6 +18,8 @@ import us.sosia.video.stream.handler.frame.FrameEncoder; +import com.xuggle.ferry.IBuffer; +import com.xuggle.xuggler.IAudioSamples; import com.xuggle.xuggler.ICodec; import com.xuggle.xuggler.IMetaData; import com.xuggle.xuggler.IPacket; @@ -29,11 +34,16 @@ public class H264StreamEncoder extends OneToOneEncoder{ protected final static Logger logger = LoggerFactory.getLogger(Logger.class); protected final IStreamCoder iStreamCoder = IStreamCoder.make(Direction.ENCODING, ICodec.ID.CODEC_ID_H264); + protected final IStreamCoder iAudioStreamCoder = IStreamCoder.make(Direction.ENCODING, ICodec.ID.CODEC_ID_AAC); protected final IPacket iPacket = IPacket.make(); protected long startTime ; protected final Dimension dimension; protected final FrameEncoder frameEncoder; + protected final AudioFormat format = new AudioFormat( + AudioFormat.Encoding.PCM_SIGNED, + 44100.0F, 16, 2, 4, 44100, false); + public H264StreamEncoder(Dimension dimension,boolean usingInternalFrameEncoder) { super(); @@ -70,6 +80,24 @@ private void initialize(){ if (revl < 0) { throw new RuntimeException("could not open the coder"); } + + iStreamCoder.setNumPicturesInGroupOfPictures(25); + + + iAudioStreamCoder.setChannels(2); + iAudioStreamCoder.setSampleRate(44100); + IRational ratea = IRational.make(44100, 1); + /* + iAudioStreamCoder.setFrameRate(ratea); + //time base + //iStreamCoder.setAutomaticallyStampPacketsForStream(true); + iAudioStreamCoder.setTimeBase(IRational.make(ratea.getDenominator(),ratea.getNumerator())); + */ + IMetaData codecOptionsa = IMetaData.make(); + revl = iAudioStreamCoder.open(codecOptionsa, null); + if (revl < 0) { + throw new RuntimeException("could not open the audio coder"); + } } @@ -79,7 +107,7 @@ protected Object encode(ChannelHandlerContext ctx, Channel channel, Object msg) throws Exception { return encode(msg); } - + public Object encode(Object msg) throws Exception { if (msg == null) { return null; @@ -105,8 +133,69 @@ public Object encode(Object msg) throws Exception { pFrame.delete(); converter.delete(); //write to the container + iPacket.setStreamIndex(0); if (iPacket.isComplete()) { + iPacket.setFlags(1); + //iPacket.delete(); + //here we send the package to the remote peer + try{ + ByteBuffer byteBuffer = iPacket.getByteBuffer(); + if (iPacket.isKeyPacket()) { + logger.info("key frame"); + } + ChannelBuffer channelBuffe = ChannelBuffers.copiedBuffer(byteBuffer.order(ByteOrder.BIG_ENDIAN)); + if (frameEncoder != null) { + System.out.println("using frame encoder"); + return channelBuffe; + //return frameEncoder.encode(channelBuffe); + } + return channelBuffe; + }finally{ + iPacket.reset(); + } + }else{ + return null; + } + } + + public Object encode( byte[] data, int numBytesRead) throws Exception { + + //System.out.println("initial data : " + Arrays.toString(data)); + //System.out.println("num bytes: " + numBytesRead + "data size: " + data.length); + + //convert the image + /* + long now = System.currentTimeMillis(); + if (startTime == 0) { + startTime = now; + } + */ + + IBuffer iBuf = IBuffer.make(null, data, 0, numBytesRead); + + IAudioSamples smp = IAudioSamples.make(iBuf,2,IAudioSamples.Format.FMT_S16); + smp.setComplete(true, numBytesRead/4, 44100, 2, IAudioSamples.Format.FMT_S16, 0); + //System.out.println("sample rate: " + smp.getSampleRate()); + iAudioStreamCoder.encodeAudio(iPacket, smp, 0); + System.out.println("packet size: " + iPacket.getSize()); + System.out.println("bytes read: " + numBytesRead); + + //write to the container + if (iPacket.isComplete()) { + /* + byte[] bufferdat = new byte[iPacket.getByteBuffer().capacity()]; + iPacket.getByteBuffer().get(bufferdat); + //System.out.print(Arrays.toString(bufferdat)); + iPacket.setStreamIndex(1); + iPacket.setFlags(2); + + //System.out.println("trying after"); + bufferdat = new byte[iPacket.getByteBuffer().capacity()]; + iPacket.getByteBuffer().get(bufferdat); + System.out.print(Arrays.toString(bufferdat)); + */ + //iPacket.delete(); //here we send the package to the remote peer try{ @@ -116,6 +205,7 @@ public Object encode(Object msg) throws Exception { } ChannelBuffer channelBuffe = ChannelBuffers.copiedBuffer(byteBuffer.order(ByteOrder.BIG_ENDIAN)); if (frameEncoder != null) { + System.out.println("using frame encoder"); return frameEncoder.encode(channelBuffe); } return channelBuffe; @@ -124,7 +214,7 @@ public Object encode(Object msg) throws Exception { iPacket.reset(); } }else{ - return null; + return encode(data,numBytesRead); } } diff --git a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/StreamFrameListener.java b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/StreamFrameListener.java index e90b50d5..f30a74f0 100644 --- a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/StreamFrameListener.java +++ b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/StreamFrameListener.java @@ -2,10 +2,14 @@ import java.awt.image.BufferedImage; +import com.xuggle.xuggler.IAudioSamples; + public interface StreamFrameListener { /** * Callback when the image is received from the live stream. * @param image The received and decoded image * */ public void onFrameReceived(BufferedImage image); + + public void onAudioRecieved(IAudioSamples samples); } From 2060fbb2edef6c5bd04fd3b93c479af51e7b5130 Mon Sep 17 00:00:00 2001 From: ryancheu Date: Mon, 24 Feb 2014 11:43:56 -0500 Subject: [PATCH 2/3] Update README.md to describe fork. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index eca9c816..9423f115 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Java Webcam Capture -This library allows you to use your build-in or external webcam directly from Java. +This library allows you to use your build-in or external webcam directly from Java. This fork adds streaming audio in addition to video. [![Build Status](https://secure.travis-ci.org/sarxos/webcam-capture.png?branch=master)](http://travis-ci.org/sarxos/webcam-capture) From dcd3af19c6e2654516966037069145b7d22e2a7a Mon Sep 17 00:00:00 2001 From: Ryan Cheu Date: Mon, 24 Feb 2014 12:10:37 -0500 Subject: [PATCH 3/3] Clean up code, indenting, remove commented out code --- .../video/stream/agent/StreamClient.java | 158 +++--- .../video/stream/agent/StreamServerAgent.java | 396 +++++++-------- .../stream/handler/H264StreamDecoder.java | 469 ++++++++---------- .../stream/handler/H264StreamEncoder.java | 331 ++++++------ .../stream/handler/StreamFrameListener.java | 2 +- 5 files changed, 631 insertions(+), 725 deletions(-) diff --git a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamClient.java b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamClient.java index 286f3270..00826bc3 100644 --- a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamClient.java +++ b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamClient.java @@ -19,91 +19,83 @@ import us.sosia.video.stream.handler.StreamFrameListener; public class StreamClient { - /** - * @author kerr - * */ - private static SourceDataLine mLine; - private static boolean isFirst = true; - private final static Dimension dimension = new Dimension(320,240); - private final static SingleVideoDisplayWindow displayWindow = new SingleVideoDisplayWindow("Stream example",dimension); - protected final static Logger logger = LoggerFactory.getLogger(StreamClient.class); - public static void main(String[] args) { - //setup the videoWindow - displayWindow.setVisible(true); - - //setup the connection - logger.info("setup dimension :{}",dimension); - StreamClientAgent clientAgent = new StreamClientAgent(new StreamFrameListenerIMPL(),dimension); - clientAgent.connect(new InetSocketAddress("localhost", 20000)); - try { - openJavaSound(); - } catch (LineUnavailableException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - - protected static class StreamFrameListenerIMPL implements StreamFrameListener{ - private volatile long count = 0; - @Override - public void onFrameReceived(BufferedImage image) { - logger.info("frame received :{}",count++); - displayWindow.updateImage(image); - } - @Override - public void onAudioRecieved(IAudioSamples samples) { - playJavaSound(samples); - // TODO Auto-generated method stub - - } - } - - private static void openJavaSound() throws LineUnavailableException + /** + * @author kerr + * */ + private static SourceDataLine mLine; + private static boolean isFirst = true; + private final static Dimension dimension = new Dimension(320,240); + private final static SingleVideoDisplayWindow displayWindow = new SingleVideoDisplayWindow("Stream example",dimension); + protected final static Logger logger = LoggerFactory.getLogger(StreamClient.class); + public static void main(String[] args) { + //setup the videoWindow + displayWindow.setVisible(true); + + //setup the connection + logger.info("setup dimension :{}",dimension); + StreamClientAgent clientAgent = new StreamClientAgent(new StreamFrameListenerIMPL(),dimension); + clientAgent.connect(new InetSocketAddress("localhost", 20000)); + try { + openJavaSound(); + } catch (LineUnavailableException e) { + e.printStackTrace(); + } + } + + + protected static class StreamFrameListenerIMPL implements StreamFrameListener{ + private volatile long count = 0; + @Override + public void onFrameReceived(BufferedImage image) { + logger.info("frame received :{}",count++); + displayWindow.updateImage(image); + } + @Override + public void onAudioRecieved(IAudioSamples samples) { + playJavaSound(samples); + } + } + + private static void openJavaSound() throws LineUnavailableException { - AudioFormat audioFormat = new AudioFormat( - AudioFormat.Encoding.PCM_SIGNED, - 44100.0F, 16, 2, 4, 44100, false); - - AudioFormat format = new AudioFormat(44100, - 16, - 2, - true, /* xuggler defaults to signed 16 bit samples */ - false); - DataLine.Info info = new DataLine.Info(SourceDataLine.class, format); - mLine = (SourceDataLine) AudioSystem.getLine(info); - /** - * if that succeeded, try opening the line. - */ - mLine.open(audioFormat); - /** - * And if that succeed, start the line. - */ - mLine.start(); - - + AudioFormat audioFormat = new AudioFormat( + AudioFormat.Encoding.PCM_SIGNED, + 44100.0F, 16, 2, 4, 44100, false); + + AudioFormat format = new AudioFormat(44100, + 16, + 2, + true, /* xuggler defaults to signed 16 bit samples */ + false); + DataLine.Info info = new DataLine.Info(SourceDataLine.class, format); + mLine = (SourceDataLine) AudioSystem.getLine(info); + /** + * if that succeeded, try opening the line. + */ + mLine.open(audioFormat); + /** + * And if that succeed, start the line. + */ + mLine.start(); + + } - - private static synchronized void playJavaSound(IAudioSamples aSamples) + + private static synchronized void playJavaSound(IAudioSamples aSamples) { - if (isFirst) { - isFirst = false; - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - /** - * We're just going to dump all the samples into the line. - */ - byte[] rawBytes = aSamples.getData().getByteArray(0, aSamples.getSize()); - System.out.println("got: " + rawBytes.length ); - mLine.write(rawBytes, 0, rawBytes.length); - - } - + if (isFirst) { + isFirst = false; + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + /** + * We're just going to dump all the samples into the line. + */ + byte[] rawBytes = aSamples.getData().getByteArray(0, aSamples.getSize()); + mLine.write(rawBytes, 0, rawBytes.length); + } } diff --git a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamServerAgent.java b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamServerAgent.java index 84a331ba..3ec96eb2 100644 --- a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamServerAgent.java +++ b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamServerAgent.java @@ -34,229 +34,213 @@ import com.github.sarxos.webcam.Webcam; public class StreamServerAgent implements IStreamServerAgent{ - protected final static Logger logger = LoggerFactory.getLogger(StreamServer.class); - protected final Webcam webcam; - protected final Dimension dimension; - protected final static ChannelGroup channelGroup = new DefaultChannelGroup(); - protected final ServerBootstrap serverBootstrap; - //I just move the stream encoder out of the channel pipeline for the performance - protected final H264StreamEncoder h264StreamEncoder; - protected final H264StreamEncoder secondEncoder; - protected volatile boolean isStreaming; - protected ScheduledExecutorService timeWorker; - protected ExecutorService encodeWorker; - protected int FPS = 25; - protected ScheduledFuture imageGrabTaskFuture; - protected TargetDataLine line; - public StreamServerAgent(Webcam webcam, Dimension dimension) { - super(); - this.webcam = webcam; - this.dimension = dimension; - //this.h264StreamEncoder = new H264StreamEncoder(dimension,false); - this.serverBootstrap = new ServerBootstrap(); - this.serverBootstrap.setFactory(new NioServerSocketChannelFactory( - Executors.newCachedThreadPool(), - Executors.newCachedThreadPool())); - this.serverBootstrap.setPipelineFactory(new StreamServerChannelPipelineFactory( - new StreamServerListenerIMPL(), - dimension)); - this.timeWorker = new ScheduledThreadPoolExecutor(1); - this.encodeWorker = Executors.newSingleThreadExecutor(); - this.h264StreamEncoder = new H264StreamEncoder(dimension, false); - this.secondEncoder = new H264StreamEncoder(dimension, false); - - AudioFormat format = new AudioFormat( - AudioFormat.Encoding.PCM_SIGNED, - 44100.0F, 16, 2, 4, 44100, false); - - this.line = null; - DataLine.Info info = new DataLine.Info(TargetDataLine.class, - - format); // format is an AudioFormat object - if (!AudioSystem.isLineSupported(info)) { - // Handle the error ... - } - // Obtain and open the line. - try { + protected final static Logger logger = LoggerFactory.getLogger(StreamServer.class); + protected final Webcam webcam; + protected final Dimension dimension; + protected final static ChannelGroup channelGroup = new DefaultChannelGroup(); + protected final ServerBootstrap serverBootstrap; + //I just move the stream encoder out of the channel pipeline for the performance + protected final H264StreamEncoder h264StreamEncoder; + protected final H264StreamEncoder secondEncoder; + protected volatile boolean isStreaming; + protected ScheduledExecutorService timeWorker; + protected ExecutorService encodeWorker; + protected int FPS = 25; + protected ScheduledFuture imageGrabTaskFuture; + protected TargetDataLine line; + public StreamServerAgent(Webcam webcam, Dimension dimension) { + super(); + this.webcam = webcam; + this.dimension = dimension; + //this.h264StreamEncoder = new H264StreamEncoder(dimension,false); + this.serverBootstrap = new ServerBootstrap(); + this.serverBootstrap.setFactory(new NioServerSocketChannelFactory( + Executors.newCachedThreadPool(), + Executors.newCachedThreadPool())); + this.serverBootstrap.setPipelineFactory(new StreamServerChannelPipelineFactory( + new StreamServerListenerIMPL(), + dimension)); + this.timeWorker = new ScheduledThreadPoolExecutor(1); + this.encodeWorker = Executors.newSingleThreadExecutor(); + this.h264StreamEncoder = new H264StreamEncoder(dimension, false); + this.secondEncoder = new H264StreamEncoder(dimension, false); + + AudioFormat format = new AudioFormat( + AudioFormat.Encoding.PCM_SIGNED, + 44100.0F, 16, 2, 4, 44100, false); + + this.line = null; + + // format is an AudioFormat object + DataLine.Info info = new DataLine.Info(TargetDataLine.class, + format); + if (!AudioSystem.isLineSupported(info)) { + System.out.println("Error, line not supported"); + } + // Obtain and open the line. + try { - this.line = (TargetDataLine) AudioSystem.getLine(info); - this.line.open(format); - } catch (LineUnavailableException ex) { - // Handle the error ... + this.line = (TargetDataLine) AudioSystem.getLine(info); + this.line.open(format); + } catch (LineUnavailableException ex) { + ex.printStackTrace(); + } + + this.line.start(); } - - this.line.start(); - } - - - - public int getFPS() { - return FPS; - } - - public void setFPS(int fPS) { - FPS = fPS; - } - - @Override - public void start(SocketAddress streamAddress) { - logger.info("Server started :{}",streamAddress); - Channel channel = serverBootstrap.bind(streamAddress); - channelGroup.add(channel); - } - - @Override - public void stop() { - logger.info("server is stoping"); - channelGroup.close(); - timeWorker.shutdown(); - encodeWorker.shutdown(); - serverBootstrap.releaseExternalResources(); - } - - private static void writeData( Object data) { - channelGroup.write(data); - /* - try { - Thread.sleep(10); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + + + + public int getFPS() { + return FPS; } - */ - } - - - private class StreamServerListenerIMPL implements StreamServerListener{ - @Override - public void onClientConnectedIn(Channel channel) { - //here we just start to stream when the first client connected in - // - channelGroup.add(channel); - if (!isStreaming) { - //do some thing - Runnable imageGrabTask = new ImageGrabTask(); - ScheduledFuture imageGrabFuture = - timeWorker.scheduleWithFixedDelay(imageGrabTask, - 0, - 1000/FPS, - TimeUnit.MILLISECONDS); - imageGrabTaskFuture = imageGrabFuture; - AudioGrabTask at = new AudioGrabTask(); - at.start(); - isStreaming = true; - } - logger.info("current connected clients :{}",channelGroup.size()); + public void setFPS(int fPS) { + FPS = fPS; } @Override - public void onClientDisconnected(Channel channel) { - channelGroup.remove(channel); - int size = channelGroup.size(); - logger.info("current connected clients :{}",size); - if (size == 1) { - //cancel the task - imageGrabTaskFuture.cancel(false); - webcam.close(); - isStreaming = false; - } + public void start(SocketAddress streamAddress) { + logger.info("Server started :{}",streamAddress); + Channel channel = serverBootstrap.bind(streamAddress); + channelGroup.add(channel); } @Override - public void onExcaption(Channel channel, Throwable t) { - channelGroup.remove(channel); - channel.close(); - int size = channelGroup.size(); - logger.info("current connected clients :{}",size); - if (size == 1) { - //cancel the task - imageGrabTaskFuture.cancel(false); - webcam.close(); - isStreaming = false; - + public void stop() { + logger.info("server is stoping"); + channelGroup.close(); + timeWorker.shutdown(); + encodeWorker.shutdown(); + serverBootstrap.releaseExternalResources(); } - - } - - protected volatile long frameCount = 0; - - private class ImageGrabTask implements Runnable{ - @Override - public void run() { - logger.info("image grabed ,count :{}",frameCount++); - BufferedImage bufferedImage = webcam.getImage(); - /** - * using this when the h264 encoder is added to the pipeline - * */ - //channelGroup.write(bufferedImage); - /** - * using this when the h264 encoder is inside this class - * */ - encodeWorker.execute(new EncodeTask(bufferedImage)); + private static void writeData( Object data) { + channelGroup.write(data); } - - } - - public class AudioGrabTask extends Thread { - public void run() { - int numBytesRead; - byte[] data = new byte [4096]; - while (true) { - if (data.length > 0 ){ - System.out.println("start read: " + System.currentTimeMillis()); - numBytesRead = line.read(data, 0, 4096); - System.out.println("end read: " + System.currentTimeMillis()); - Object msg2 = null; - try { - if ( numBytesRead > 0 ) { - msg2 = secondEncoder.encode(data,numBytesRead); + + + private class StreamServerListenerIMPL implements StreamServerListener{ + + @Override + public void onClientConnectedIn(Channel channel) { + //here we just start to stream when the first client connected in + channelGroup.add(channel); + if (!isStreaming) { + //do some thing + Runnable imageGrabTask = new ImageGrabTask(); + ScheduledFuture imageGrabFuture = + timeWorker.scheduleWithFixedDelay(imageGrabTask, + 0, + 1000/FPS, + TimeUnit.MILLISECONDS); + imageGrabTaskFuture = imageGrabFuture; + AudioGrabTask at = new AudioGrabTask(); + at.start(); + isStreaming = true; } - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - if ( msg2 != null ) { - System.out.println("writing audio"); - writeData(msg2); - } - Thread.yield(); + logger.info("current connected clients :{}",channelGroup.size()); } - } - } - } - - - private class EncodeTask implements Runnable{ - private final BufferedImage image; - - public EncodeTask(BufferedImage image) { - super(); - this.image = image; - } - @Override - public void run() { - try { - Object msg = h264StreamEncoder.encode(image); - if (msg != null) { - writeData(msg); + @Override + public void onClientDisconnected(Channel channel) { + channelGroup.remove(channel); + int size = channelGroup.size(); + logger.info("current connected clients :{}",size); + if (size == 1) { + //cancel the task + imageGrabTaskFuture.cancel(false); + webcam.close(); + isStreaming = false; + } } - - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - } - - } - - - - + @Override + public void onExcaption(Channel channel, Throwable t) { + channelGroup.remove(channel); + channel.close(); + int size = channelGroup.size(); + logger.info("current connected clients :{}",size); + if (size == 1) { + //cancel the task + imageGrabTaskFuture.cancel(false); + webcam.close(); + isStreaming = false; + + } + + } + + protected volatile long frameCount = 0; + + private class ImageGrabTask implements Runnable{ + + @Override + public void run() { + logger.info("image grabed ,count :{}",frameCount++); + BufferedImage bufferedImage = webcam.getImage(); + /** + * using this when the h264 encoder is added to the pipeline + * */ + //channelGroup.write(bufferedImage); + /** + * using this when the h264 encoder is inside this class + * */ + encodeWorker.execute(new EncodeTask(bufferedImage)); + } + } + + public class AudioGrabTask extends Thread { + public void run() { + int numBytesRead; + byte[] data = new byte [4096]; + while (true) { + if (data.length > 0 ){ + System.out.println("start read: " + System.currentTimeMillis()); + numBytesRead = line.read(data, 0, 4096); + System.out.println("end read: " + System.currentTimeMillis()); + Object msg2 = null; + try { + if ( numBytesRead > 0 ) { + msg2 = secondEncoder.encode(data,numBytesRead); + } + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + if ( msg2 != null ) { + System.out.println("writing audio"); + writeData(msg2); + } + Thread.yield(); + } + } + } + } + + + private class EncodeTask implements Runnable{ + private final BufferedImage image; + + public EncodeTask(BufferedImage image) { + super(); + this.image = image; + } + + @Override + public void run() { + try { + Object msg = h264StreamEncoder.encode(image); + if (msg != null) { + writeData(msg); + } + + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + } } diff --git a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamDecoder.java b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamDecoder.java index 5be1a8b1..194756a7 100644 --- a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamDecoder.java +++ b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamDecoder.java @@ -29,276 +29,245 @@ * This codec will encode the bufferedImage to h264 stream * **/ public class H264StreamDecoder extends OneToOneDecoder{ - protected final static Logger logger = LoggerFactory.getLogger(H264StreamDecoder.class); - protected final IStreamCoder iStreamCoder = IStreamCoder.make(Direction.DECODING, ICodec.ID.CODEC_ID_H264); - protected final IStreamCoder iAudioStreamCoder = IStreamCoder.make(Direction.DECODING, ICodec.ID.CODEC_ID_AAC); - protected final Type type = ConverterFactory.findRegisteredConverter(ConverterFactory.XUGGLER_BGR_24); - protected final StreamFrameListener streamFrameListener; - protected final Dimension dimension; + protected final static Logger logger = LoggerFactory.getLogger(H264StreamDecoder.class); + protected final IStreamCoder iStreamCoder = IStreamCoder.make(Direction.DECODING, ICodec.ID.CODEC_ID_H264); + protected final IStreamCoder iAudioStreamCoder = IStreamCoder.make(Direction.DECODING, ICodec.ID.CODEC_ID_AAC); + protected final Type type = ConverterFactory.findRegisteredConverter(ConverterFactory.XUGGLER_BGR_24); + protected final StreamFrameListener streamFrameListener; + protected final Dimension dimension; - protected final FrameDecoder frameDecoder; - protected final ExecutorService decodeWorker ; - /** - * - * Cause there may be one or more image in the frame,so we need an Stream listener here to get all the image - * - * */ - - - - - - public H264StreamDecoder(StreamFrameListener streamFrameListener, - Dimension dimension,boolean internalFrameDecoder,boolean decodeInOtherThread) { - super(); - this.streamFrameListener = streamFrameListener; - this.dimension = dimension; - if (internalFrameDecoder) { - frameDecoder = new FrameDecoder(4); - }else { - frameDecoder = null; - } - if (decodeInOtherThread) { - decodeWorker = Executors.newSingleThreadExecutor(); - }else { - decodeWorker = null; - } + protected final FrameDecoder frameDecoder; + protected final ExecutorService decodeWorker ; + /** + * + * Cause there may be one or more image in the frame,so we need an Stream listener here to get all the image + * + * */ + public H264StreamDecoder(StreamFrameListener streamFrameListener, + Dimension dimension,boolean internalFrameDecoder,boolean decodeInOtherThread) { + super(); + this.streamFrameListener = streamFrameListener; + this.dimension = dimension; + if (internalFrameDecoder) { + frameDecoder = new FrameDecoder(4); + }else { + frameDecoder = null; + } + if (decodeInOtherThread) { + decodeWorker = Executors.newSingleThreadExecutor(); + }else { + decodeWorker = null; + } - initialize(); - } - + initialize(); + } + private void initialize(){ + iStreamCoder.open(null, null); + iAudioStreamCoder.open(null, null); + } - private void initialize(){ - //iStreamCoder.setNumPicturesInGroupOfPictures(20); - //iStreamCoder.setBitRate(250000); - //iStreamCoder.setBitRateTolerance(9000); - //iStreamCoder.setPixelType(IPixelFormat.Type.YUV420P); - //iStreamCoder.setHeight(dimension.height); - //iStreamCoder.setWidth(dimension.width); - //iStreamCoder.setFlag(IStreamCoder.Flags.FLAG_QSCALE, true); - //iStreamCoder.setGlobalQuality(0); - //rate - //IRational rate = IRational.make(25, 1); - //iStreamCoder.setFrameRate(rate); - //time base - //iStreamCoder.setAutomaticallyStampPacketsForStream(true); - //iStreamCoder.setTimeBase(IRational.make(rate.getDenominator(),rate.getNumerator())); - iStreamCoder.open(null, null); - iAudioStreamCoder.open(null, null); - } - - + @Override + protected Object decode(ChannelHandlerContext ctx, Channel channel, + final Object msg) throws Exception { + if (decodeWorker != null) { + decodeWorker.execute(new decodeTask(msg)); + return null; + } + else { + if (msg == null) { + throw new NullPointerException("you cannot pass into an null to the decode"); + } + ChannelBuffer frameBuffer; + if (frameDecoder != null) { + frameBuffer = frameDecoder.decode((ChannelBuffer)msg); + if (frameBuffer == null) { + return null; + } + } + else { + frameBuffer = (ChannelBuffer)msg; + } - - - @Override - protected Object decode(ChannelHandlerContext ctx, Channel channel, - final Object msg) throws Exception { - if (decodeWorker != null) { - decodeWorker.execute(new decodeTask(msg)); - return null; - }else { - if (msg == null) { - throw new NullPointerException("you cannot pass into an null to the decode"); - } - ChannelBuffer frameBuffer; - if (frameDecoder != null) { - frameBuffer = frameDecoder.decode((ChannelBuffer)msg); - if (frameBuffer == null) { - return null; - } - - }else { - frameBuffer = (ChannelBuffer)msg; - } - - - - int size = frameBuffer.readableBytes(); - logger.info("decode the frame size :{}",size); - - //start to decode - IBuffer iBuffer = IBuffer.make(null, size); - IPacket iPacket = IPacket.make(iBuffer); - iPacket.getByteBuffer().put(frameBuffer.toByteBuffer()); - //decode the packet - if (!iPacket.isComplete()) { - return null; - } - logger.info("packet stream index: " + iPacket.getFlags()); + int size = frameBuffer.readableBytes(); + logger.info("decode the frame size :{}",size); - if ( iPacket.getByteBuffer().get() != -1 ) { + //start to decode + IBuffer iBuffer = IBuffer.make(null, size); + IPacket iPacket = IPacket.make(iBuffer); + iPacket.getByteBuffer().put(frameBuffer.toByteBuffer()); + //decode the packet + if (!iPacket.isComplete()) { + return null; + } + logger.info("packet stream index: " + iPacket.getFlags()); - IVideoPicture picture = IVideoPicture.make(IPixelFormat.Type.YUV420P, - dimension.width, dimension.height); - try { - // decode the packet into the video picture - int postion = 0; - int packageSize = iPacket.getSize(); - while(postion < packageSize){ - postion+= iStreamCoder.decodeVideo(picture, iPacket, postion); - if (postion < 0) - throw new RuntimeException("error " - + " decoding video"); - // if this is a complete picture, dispatch the picture - if (picture.isComplete()){ - IConverter converter = ConverterFactory.createConverter(type - .getDescriptor(), picture); - BufferedImage image = converter.toImage(picture); - //BufferedImage convertedImage = ImageUtils.convertToType(image, BufferedImage.TYPE_3BYTE_BGR); - //here ,put out the image - if (streamFrameListener != null) { - streamFrameListener.onFrameReceived(image); - } - converter.delete(); - }else{ - picture.delete(); - iPacket.delete(); - return null; - } - //clean the picture and reuse it - picture.getByteBuffer().clear(); - } - } finally { - if (picture != null) - picture.delete(); - iPacket.delete(); - // ByteBufferUtil.destroy(data); - } - } - else { - //System.out.println("decoding audio"); - IAudioSamples samples = IAudioSamples.make(1024, 1); + if ( iPacket.getByteBuffer().get() != -1 ) { + IVideoPicture picture = IVideoPicture.make(IPixelFormat.Type.YUV420P, + dimension.width, dimension.height); + try { + // decode the packet into the video picture + int postion = 0; + int packageSize = iPacket.getSize(); + while(postion < packageSize){ + postion+= iStreamCoder.decodeVideo(picture, iPacket, postion); + if (postion < 0) + throw new RuntimeException("error " + + " decoding video"); + // if this is a complete picture, dispatch the picture + if (picture.isComplete()){ + IConverter converter = ConverterFactory.createConverter(type + .getDescriptor(), picture); + BufferedImage image = converter.toImage(picture); + //BufferedImage convertedImage = ImageUtils.convertToType(image, BufferedImage.TYPE_3BYTE_BGR); + //here ,put out the image + if (streamFrameListener != null) { + streamFrameListener.onFrameReceived(image); + } + converter.delete(); + } + else { + picture.delete(); + iPacket.delete(); + return null; + } + //clean the picture and reuse it + picture.getByteBuffer().clear(); + } + } finally { + if (picture != null) + picture.delete(); + iPacket.delete(); + // ByteBufferUtil.destroy(data); + } + } + else { + IAudioSamples samples = IAudioSamples.make(1024, 1); - /* - * A packet can actually contain multiple sets of samples (or frames of samples - * in audio-decoding speak). So, we may need to call decode audio multiple - * times at different offsets in the packet's data. We capture that here. - */ - int offset = 0; + /* + * A packet can actually contain multiple sets of samples (or frames of samples + * in audio-decoding speak). So, we may need to call decode audio multiple + * times at different offsets in the packet's data. We capture that here. + */ + int offset = 0; - /* - * Keep going until we've processed all data - */ - while(offset < iPacket.getSize()) - { - //System.out.println("starting decode"); - int bytesDecoded = iAudioStreamCoder.decodeAudio(samples, iPacket, offset); - if (bytesDecoded < 0) - throw new RuntimeException("got error decoding audio in stream"); + /* + * Keep going until we've processed all data + */ + while(offset < iPacket.getSize()) { + int bytesDecoded = iAudioStreamCoder.decodeAudio(samples, iPacket, offset); + if (bytesDecoded < 0) + throw new RuntimeException("got error decoding audio in stream"); - System.out.println("decoded :" + bytesDecoded); - offset += bytesDecoded; + offset += bytesDecoded; - //System.out.println("decoded some bytes"); - /* - * Some decoder will consume data in a packet, but will not be able to construct - * a full set of samples yet. Therefore you should always check if you - * got a complete set of samples from the decoder - */ - if (samples.isComplete()) - { - if (streamFrameListener != null) { - streamFrameListener.onAudioRecieved(samples); - } - } - } - - System.out.println("got audio frame"); - } - return null; - } - } + /* + * Some decoder will consume data in a packet, but will not be able to construct + * a full set of samples yet. Therefore you should always check if you + * got a complete set of samples from the decoder + */ + if (samples.isComplete()) { + if (streamFrameListener != null) { + streamFrameListener.onAudioRecieved(samples); + } + } + } + } + return null; + } + } - private class decodeTask implements Runnable{ - private final Object msg; + private class decodeTask implements Runnable{ + private final Object msg; - public decodeTask(Object msg) { - super(); - this.msg = msg; - } + public decodeTask(Object msg) { + super(); + this.msg = msg; + } - @Override - public void run() { - if (msg == null) { - return; - } - ChannelBuffer frameBuffer; - if (frameDecoder != null) { - try { - frameBuffer = frameDecoder.decode((ChannelBuffer)msg); - if (frameBuffer == null) { - return ; - } - } catch (Exception e) { - return; - } + @Override + public void run() { + if (msg == null) { + return; + } + ChannelBuffer frameBuffer; + if (frameDecoder != null) { + try { + frameBuffer = frameDecoder.decode((ChannelBuffer)msg); + if (frameBuffer == null) { + return ; + } + } catch (Exception e) { + return; + } - }else { - frameBuffer = (ChannelBuffer)msg; - } + } + else { + frameBuffer = (ChannelBuffer)msg; + } - int size = frameBuffer.readableBytes(); - logger.info("decode the frame size :{}",size); - //start to decode - IBuffer iBuffer = IBuffer.make(null, size); - IPacket iPacket = IPacket.make(iBuffer); - iPacket.getByteBuffer().put(frameBuffer.toByteBuffer()); - //decode the packet - if (!iPacket.isComplete()) { - return ; - } - logger.info("packet stream index: " + iPacket.getStreamIndex()); - if ( iPacket.getStreamIndex() == 0 ) { - IVideoPicture picture = IVideoPicture.make(IPixelFormat.Type.YUV420P, - dimension.width, dimension.height); - try { - // decode the packet into the video picture - int postion = 0; - int packageSize = iPacket.getSize(); - while(postion < packageSize){ - postion+= iStreamCoder.decodeVideo(picture, iPacket, postion); - if (postion < 0) - throw new RuntimeException("error " - + " decoding video"); - // if this is a complete picture, dispatch the picture - if (picture.isComplete()){ - IConverter converter = ConverterFactory.createConverter(type - .getDescriptor(), picture); - BufferedImage image = converter.toImage(picture); - //BufferedImage convertedImage = ImageUtils.convertToType(image, BufferedImage.TYPE_3BYTE_BGR); - //here ,put out the image - if (streamFrameListener != null) { - streamFrameListener.onFrameReceived(image); - } - converter.delete(); - }else{ - picture.delete(); - iPacket.delete(); - return ; - } - //clean the picture and reuse it - picture.getByteBuffer().clear(); - } - } finally { - if (picture != null) - picture.delete(); - iPacket.delete(); - // ByteBufferUtil.destroy(data); - } - return ; - } - else { - iPacket.delete(); - logger.info("got audio frame"); - } - } + int size = frameBuffer.readableBytes(); + logger.info("decode the frame size :{}",size); + //start to decode + IBuffer iBuffer = IBuffer.make(null, size); + IPacket iPacket = IPacket.make(iBuffer); + iPacket.getByteBuffer().put(frameBuffer.toByteBuffer()); + //decode the packet + if (!iPacket.isComplete()) { + return ; + } + logger.info("packet stream index: " + iPacket.getStreamIndex()); + if ( iPacket.getStreamIndex() == 0 ) { + IVideoPicture picture = IVideoPicture.make(IPixelFormat.Type.YUV420P, + dimension.width, dimension.height); + try { + // decode the packet into the video picture + int postion = 0; + int packageSize = iPacket.getSize(); + while(postion < packageSize){ + postion+= iStreamCoder.decodeVideo(picture, iPacket, postion); + if (postion < 0) { + throw new RuntimeException("error " + + " decoding video"); + } + // if this is a complete picture, dispatch the picture + if (picture.isComplete()){ + IConverter converter = ConverterFactory.createConverter(type + .getDescriptor(), picture); + BufferedImage image = converter.toImage(picture); + //BufferedImage convertedImage = ImageUtils.convertToType(image, BufferedImage.TYPE_3BYTE_BGR); + //here ,put out the image + if (streamFrameListener != null) { + streamFrameListener.onFrameReceived(image); + } + converter.delete(); + } + else { + picture.delete(); + iPacket.delete(); + return ; + } + //clean the picture and reuse it + picture.getByteBuffer().clear(); + } + } finally { + if (picture != null) { + picture.delete(); + } + iPacket.delete(); + } + return ; + } + else { + iPacket.delete(); + logger.info("got audio frame"); + } + } - } + } diff --git a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamEncoder.java b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamEncoder.java index b5bd3f15..f44b1087 100644 --- a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamEncoder.java +++ b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamEncoder.java @@ -32,190 +32,151 @@ import com.xuggle.xuggler.video.IConverter; public class H264StreamEncoder extends OneToOneEncoder{ - protected final static Logger logger = LoggerFactory.getLogger(Logger.class); - protected final IStreamCoder iStreamCoder = IStreamCoder.make(Direction.ENCODING, ICodec.ID.CODEC_ID_H264); - protected final IStreamCoder iAudioStreamCoder = IStreamCoder.make(Direction.ENCODING, ICodec.ID.CODEC_ID_AAC); - protected final IPacket iPacket = IPacket.make(); - protected long startTime ; - protected final Dimension dimension; - protected final FrameEncoder frameEncoder; - - protected final AudioFormat format = new AudioFormat( - AudioFormat.Encoding.PCM_SIGNED, - 44100.0F, 16, 2, 4, 44100, false); - - - public H264StreamEncoder(Dimension dimension,boolean usingInternalFrameEncoder) { - super(); - this.dimension = dimension; - if (usingInternalFrameEncoder) { - frameEncoder = new FrameEncoder(4); - }else { - frameEncoder = null; - } - initialize(); - } - - private void initialize(){ - //setup - iStreamCoder.setNumPicturesInGroupOfPictures(25); - - iStreamCoder.setBitRate(200000); - iStreamCoder.setBitRateTolerance(10000); - iStreamCoder.setPixelType(Type.YUV420P); - iStreamCoder.setHeight(dimension.height); - iStreamCoder.setWidth(dimension.width); - iStreamCoder.setFlag(IStreamCoder.Flags.FLAG_QSCALE, true); - iStreamCoder.setGlobalQuality(0); - //rate - IRational rate = IRational.make(25, 1); - iStreamCoder.setFrameRate(rate); - //time base - //iStreamCoder.setAutomaticallyStampPacketsForStream(true); - iStreamCoder.setTimeBase(IRational.make(rate.getDenominator(),rate.getNumerator())); - IMetaData codecOptions = IMetaData.make(); - codecOptions.setValue("tune", "zerolatency");// equals -tune zerolatency in ffmpeg - //open it - int revl = iStreamCoder.open(codecOptions, null); - if (revl < 0) { - throw new RuntimeException("could not open the coder"); - } - - iStreamCoder.setNumPicturesInGroupOfPictures(25); - - - iAudioStreamCoder.setChannels(2); - iAudioStreamCoder.setSampleRate(44100); - IRational ratea = IRational.make(44100, 1); - /* - iAudioStreamCoder.setFrameRate(ratea); - //time base - //iStreamCoder.setAutomaticallyStampPacketsForStream(true); - iAudioStreamCoder.setTimeBase(IRational.make(ratea.getDenominator(),ratea.getNumerator())); - */ - IMetaData codecOptionsa = IMetaData.make(); - revl = iAudioStreamCoder.open(codecOptionsa, null); - if (revl < 0) { - throw new RuntimeException("could not open the audio coder"); - } - } - - - - @Override - protected Object encode(ChannelHandlerContext ctx, Channel channel, - Object msg) throws Exception { - return encode(msg); - } - - public Object encode(Object msg) throws Exception { - if (msg == null) { - return null; - } - if (!(msg instanceof BufferedImage)) { - throw new IllegalArgumentException("your need to pass into an bufferedimage"); - } - logger.info("encode the frame"); - BufferedImage bufferedImage = (BufferedImage)msg; - //here is the encode - //convert the image - BufferedImage convetedImage = ImageUtils.convertToType(bufferedImage, BufferedImage.TYPE_3BYTE_BGR); - IConverter converter = ConverterFactory.createConverter(convetedImage, Type.YUV420P); - //to frame - long now = System.currentTimeMillis(); - if (startTime == 0) { - startTime = now; - } - IVideoPicture pFrame = converter.toPicture(convetedImage, (now - startTime)*1000); - //pFrame.setQuality(0); - iStreamCoder.encodeVideo(iPacket, pFrame, 0) ; - //free the MEM - pFrame.delete(); - converter.delete(); - //write to the container - iPacket.setStreamIndex(0); - if (iPacket.isComplete()) { - iPacket.setFlags(1); - //iPacket.delete(); - //here we send the package to the remote peer - try{ - ByteBuffer byteBuffer = iPacket.getByteBuffer(); - if (iPacket.isKeyPacket()) { - logger.info("key frame"); - } - ChannelBuffer channelBuffe = ChannelBuffers.copiedBuffer(byteBuffer.order(ByteOrder.BIG_ENDIAN)); - if (frameEncoder != null) { - System.out.println("using frame encoder"); - return channelBuffe; - //return frameEncoder.encode(channelBuffe); - } - return channelBuffe; - - }finally{ - iPacket.reset(); - } - }else{ - return null; - } - } - - public Object encode( byte[] data, int numBytesRead) throws Exception { - - //System.out.println("initial data : " + Arrays.toString(data)); - //System.out.println("num bytes: " + numBytesRead + "data size: " + data.length); - - //convert the image - /* - long now = System.currentTimeMillis(); - if (startTime == 0) { - startTime = now; - } - */ - - IBuffer iBuf = IBuffer.make(null, data, 0, numBytesRead); - - IAudioSamples smp = IAudioSamples.make(iBuf,2,IAudioSamples.Format.FMT_S16); - smp.setComplete(true, numBytesRead/4, 44100, 2, IAudioSamples.Format.FMT_S16, 0); - //System.out.println("sample rate: " + smp.getSampleRate()); - iAudioStreamCoder.encodeAudio(iPacket, smp, 0); - System.out.println("packet size: " + iPacket.getSize()); - System.out.println("bytes read: " + numBytesRead); - - //write to the container - if (iPacket.isComplete()) { - /* - byte[] bufferdat = new byte[iPacket.getByteBuffer().capacity()]; - iPacket.getByteBuffer().get(bufferdat); - //System.out.print(Arrays.toString(bufferdat)); - iPacket.setStreamIndex(1); - iPacket.setFlags(2); - - //System.out.println("trying after"); - bufferdat = new byte[iPacket.getByteBuffer().capacity()]; - iPacket.getByteBuffer().get(bufferdat); - System.out.print(Arrays.toString(bufferdat)); - */ - - //iPacket.delete(); - //here we send the package to the remote peer - try{ - ByteBuffer byteBuffer = iPacket.getByteBuffer(); - if (iPacket.isKeyPacket()) { - logger.info("key frame"); - } - ChannelBuffer channelBuffe = ChannelBuffers.copiedBuffer(byteBuffer.order(ByteOrder.BIG_ENDIAN)); - if (frameEncoder != null) { - System.out.println("using frame encoder"); - return frameEncoder.encode(channelBuffe); - } - return channelBuffe; - - }finally{ - iPacket.reset(); - } - }else{ - return encode(data,numBytesRead); - } - } + protected final static Logger logger = LoggerFactory.getLogger(Logger.class); + protected final IStreamCoder iStreamCoder = IStreamCoder.make(Direction.ENCODING, ICodec.ID.CODEC_ID_H264); + protected final IStreamCoder iAudioStreamCoder = IStreamCoder.make(Direction.ENCODING, ICodec.ID.CODEC_ID_AAC); + protected final IPacket iPacket = IPacket.make(); + protected long startTime ; + protected final Dimension dimension; + protected final FrameEncoder frameEncoder; + + protected final AudioFormat format = new AudioFormat( + AudioFormat.Encoding.PCM_SIGNED, + 44100.0F, 16, 2, 4, 44100, false); + + + public H264StreamEncoder(Dimension dimension,boolean usingInternalFrameEncoder) { + super(); + this.dimension = dimension; + if (usingInternalFrameEncoder) { + frameEncoder = new FrameEncoder(4); + }else { + frameEncoder = null; + } + initialize(); + } + + private void initialize(){ + //setup + iStreamCoder.setNumPicturesInGroupOfPictures(25); + + iStreamCoder.setBitRate(200000); + iStreamCoder.setBitRateTolerance(10000); + iStreamCoder.setPixelType(Type.YUV420P); + iStreamCoder.setHeight(dimension.height); + iStreamCoder.setWidth(dimension.width); + iStreamCoder.setFlag(IStreamCoder.Flags.FLAG_QSCALE, true); + iStreamCoder.setGlobalQuality(0); + //rate + IRational rate = IRational.make(25, 1); + iStreamCoder.setFrameRate(rate); + //time base + iStreamCoder.setTimeBase(IRational.make(rate.getDenominator(),rate.getNumerator())); + IMetaData codecOptions = IMetaData.make(); + codecOptions.setValue("tune", "zerolatency");// equals -tune zerolatency in ffmpeg + //open it + int revl = iStreamCoder.open(codecOptions, null); + if (revl < 0) { + throw new RuntimeException("could not open the coder"); + } + + iStreamCoder.setNumPicturesInGroupOfPictures(25); + + + iAudioStreamCoder.setChannels(2); + iAudioStreamCoder.setSampleRate(44100); + IRational ratea = IRational.make(44100, 1); + IMetaData codecOptionsa = IMetaData.make(); + revl = iAudioStreamCoder.open(codecOptionsa, null); + if (revl < 0) { + throw new RuntimeException("could not open the audio coder"); + } + } + + + + @Override + protected Object encode(ChannelHandlerContext ctx, Channel channel, + Object msg) throws Exception { + return encode(msg); + } + + public Object encode(Object msg) throws Exception { + if (msg == null) { + return null; + } + if (!(msg instanceof BufferedImage)) { + throw new IllegalArgumentException("your need to pass into an bufferedimage"); + } + logger.info("encode the frame"); + BufferedImage bufferedImage = (BufferedImage)msg; + //here is the encode + //convert the image + BufferedImage convetedImage = ImageUtils.convertToType(bufferedImage, BufferedImage.TYPE_3BYTE_BGR); + IConverter converter = ConverterFactory.createConverter(convetedImage, Type.YUV420P); + //to frame + long now = System.currentTimeMillis(); + if (startTime == 0) { + startTime = now; + } + IVideoPicture pFrame = converter.toPicture(convetedImage, (now - startTime)*1000); + iStreamCoder.encodeVideo(iPacket, pFrame, 0) ; + //free the MEM + pFrame.delete(); + converter.delete(); + //write to the container + iPacket.setStreamIndex(0); + if (iPacket.isComplete()) { + iPacket.setFlags(1); + //here we send the package to the remote peer + try{ + ByteBuffer byteBuffer = iPacket.getByteBuffer(); + if (iPacket.isKeyPacket()) { + logger.info("key frame"); + } + ChannelBuffer channelBuffe = ChannelBuffers.copiedBuffer(byteBuffer.order(ByteOrder.BIG_ENDIAN)); + if (frameEncoder != null) { + return channelBuffe; + } + return channelBuffe; + + }finally{ + iPacket.reset(); + } + }else{ + return null; + } + } + + public Object encode( byte[] data, int numBytesRead) throws Exception { + IBuffer iBuf = IBuffer.make(null, data, 0, numBytesRead); + + IAudioSamples smp = IAudioSamples.make(iBuf,2,IAudioSamples.Format.FMT_S16); + smp.setComplete(true, numBytesRead/4, 44100, 2, IAudioSamples.Format.FMT_S16, 0); + iAudioStreamCoder.encodeAudio(iPacket, smp, 0); + + //write to the container + if (iPacket.isComplete()) { + //here we send the package to the remote peer + try{ + ByteBuffer byteBuffer = iPacket.getByteBuffer(); + if (iPacket.isKeyPacket()) { + logger.info("key frame"); + } + ChannelBuffer channelBuffe = ChannelBuffers.copiedBuffer(byteBuffer.order(ByteOrder.BIG_ENDIAN)); + if (frameEncoder != null) { + return frameEncoder.encode(channelBuffe); + } + return channelBuffe; + + } + finally { + iPacket.reset(); + } + } + else { + return encode(data,numBytesRead); + } + } } diff --git a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/StreamFrameListener.java b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/StreamFrameListener.java index f30a74f0..fd39712f 100644 --- a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/StreamFrameListener.java +++ b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/StreamFrameListener.java @@ -10,6 +10,6 @@ public interface StreamFrameListener { * @param image The received and decoded image * */ public void onFrameReceived(BufferedImage image); - + public void onAudioRecieved(IAudioSamples samples); }