for #250, merge avc to codec. use queue to dequeue.

@@ -366,7 +366,7 @@ ModuleLibIncs=(${SRS_OBJS_DIR})
 MODULE_FILES=("srs_kernel_error" "srs_kernel_log" "srs_kernel_stream"
 MODULE_FILES=("srs_kernel_error" "srs_kernel_log" "srs_kernel_stream"
         "srs_kernel_utility" "srs_kernel_flv" "srs_kernel_codec" "srs_kernel_file" 
         "srs_kernel_utility" "srs_kernel_flv" "srs_kernel_codec" "srs_kernel_file" 
         "srs_kernel_consts" "srs_kernel_aac" "srs_kernel_mp3" "srs_kernel_ts"
         "srs_kernel_consts" "srs_kernel_aac" "srs_kernel_mp3" "srs_kernel_ts"
-        "srs_kernel_avc" "srs_kernel_buffer")
+        "srs_kernel_buffer")
 KERNEL_INCS="src/kernel"; MODULE_DIR=${KERNEL_INCS} . auto/modules.sh
 KERNEL_INCS="src/kernel"; MODULE_DIR=${KERNEL_INCS} . auto/modules.sh
 KERNEL_OBJS="${MODULE_OBJS[@]}"
 KERNEL_OBJS="${MODULE_OBJS[@]}"
 #
 #

@@ -20,8 +20,6 @@ file
 	kernel readonly separator,
 	kernel readonly separator,
 	../../src/kernel/srs_kernel_aac.hpp,
 	../../src/kernel/srs_kernel_aac.hpp,
 	../../src/kernel/srs_kernel_aac.cpp,
 	../../src/kernel/srs_kernel_aac.cpp,
-	../../src/kernel/srs_kernel_avc.hpp,
-	../../src/kernel/srs_kernel_avc.cpp,
 	../../src/kernel/srs_kernel_buffer.hpp,
 	../../src/kernel/srs_kernel_buffer.hpp,
 	../../src/kernel/srs_kernel_buffer.cpp,
 	../../src/kernel/srs_kernel_buffer.cpp,
 	../../src/kernel/srs_kernel_codec.hpp,
 	../../src/kernel/srs_kernel_codec.hpp,

@@ -47,7 +47,7 @@ using namespace std;
 #include <srs_rtmp_sdk.hpp>
 #include <srs_rtmp_sdk.hpp>
 #include <srs_app_pithy_print.hpp>
 #include <srs_app_pithy_print.hpp>
 #include <srs_kernel_utility.hpp>
 #include <srs_kernel_utility.hpp>
-#include <srs_kernel_avc.hpp>
+#include <srs_kernel_codec.hpp>
 #include <srs_kernel_file.hpp>
 #include <srs_kernel_file.hpp>
 #include <srs_rtmp_buffer.hpp>
 #include <srs_rtmp_buffer.hpp>
 #include <srs_kernel_ts.hpp>
 #include <srs_kernel_ts.hpp>

@@ -57,6 +57,68 @@ ISrsUdpHandler::~ISrsUdpHandler()
 {
 {
 }
 }
 
 
+SrsMpegtsQueue::SrsMpegtsQueue()
+{
+    nb_audios = nb_videos = 0;
+}
+
+SrsMpegtsQueue::~SrsMpegtsQueue()
+{
+    std::map<int64_t, SrsSharedPtrMessage*>::iterator it;
+    for (it = msgs.begin(); it != msgs.end(); ++it) {
+        SrsSharedPtrMessage* msg = it->second;
+        srs_freep(msg);
+    }
+    msgs.clear();
+}
+
+int SrsMpegtsQueue::push(SrsSharedPtrMessage* msg)
+{
+    int ret = ERROR_SUCCESS;
+
+    if (msgs.find(msg->timestamp) != msgs.end()) {
+        srs_warn("mpegts: free the msg for dts exists, dts=%"PRId64, msg->timestamp);
+        srs_freep(msg);
+        return ret;
+    }
+
+    if (msg->is_audio()) {
+        nb_audios++;
+    }
+
+    if (msg->is_video()) {
+        nb_videos++;
+    }
+
+    msgs[msg->timestamp] = msg;
+
+    return ret;
+}
+
+SrsSharedPtrMessage* SrsMpegtsQueue::dequeue()
+{
+    // got 2+ videos and audios, ok to dequeue.
+    bool av_ok = nb_videos >= 2 && nb_audios >= 2;
+    // 100 videos about 30s, while 300 audios about 30s
+    bool av_overflow = nb_videos > 100 || nb_audios > 300;
+
+    if (av_ok || av_overflow) {
+        std::map<int64_t, SrsSharedPtrMessage*>::iterator it = msgs.begin();
+        SrsSharedPtrMessage* msg = it->second;
+        msgs.erase(it);
+
+        if (msg->is_audio()) {
+            nb_audios--;
+        }
+
+        if (msg->is_video()) {
+            nb_videos--;
+        }
+    }
+
+    return NULL;
+}
+
 SrsMpegtsOverUdp::SrsMpegtsOverUdp(SrsConfDirective* c)
 SrsMpegtsOverUdp::SrsMpegtsOverUdp(SrsConfDirective* c)
 {
 {
     stream = new SrsStream();
     stream = new SrsStream();
@@ -72,6 +134,7 @@ SrsMpegtsOverUdp::SrsMpegtsOverUdp(SrsConfDirective* c)
     h264_sps_changed = false;
     h264_sps_changed = false;
     h264_pps_changed = false;
     h264_pps_changed = false;
     h264_sps_pps_sent = false;
     h264_sps_pps_sent = false;
+    queue = new SrsMpegtsQueue();
 }
 }
 
 
 SrsMpegtsOverUdp::~SrsMpegtsOverUdp()
 SrsMpegtsOverUdp::~SrsMpegtsOverUdp()
@@ -82,6 +145,7 @@ SrsMpegtsOverUdp::~SrsMpegtsOverUdp()
     srs_freep(stream);
     srs_freep(stream);
     srs_freep(context);
     srs_freep(context);
     srs_freep(avc);
     srs_freep(avc);
+    srs_freep(queue);
 }
 }
 
 
 int SrsMpegtsOverUdp::on_udp_packet(sockaddr_in* from, char* buf, int nb_buf)
 int SrsMpegtsOverUdp::on_udp_packet(sockaddr_in* from, char* buf, int nb_buf)
@@ -280,11 +344,14 @@ int SrsMpegtsOverUdp::on_ts_video(SrsTsMessage* msg, SrsStream* avs)
 
 
         // it may be return error, but we must process all packets.
         // it may be return error, but we must process all packets.
         if ((ret = write_h264_raw_frame(frame, frame_size, dts, pts)) != ERROR_SUCCESS) {
         if ((ret = write_h264_raw_frame(frame, frame_size, dts, pts)) != ERROR_SUCCESS) {
-            if (ret = ERROR_H264_DROP_BEFORE_SPS_PPS) {
+            if (ret == ERROR_H264_DROP_BEFORE_SPS_PPS) {
                 continue;
                 continue;
             }
             }
             return ret;
             return ret;
         }
         }
+
+        // for video, drop others with same pts/dts.
+        break;
     }
     }
 
 
     return ret;
     return ret;
@@ -399,15 +466,28 @@ int SrsMpegtsOverUdp::rtmp_write_packet(char type, u_int32_t timestamp, char* da
     SrsSharedPtrMessage* msg = NULL;
     SrsSharedPtrMessage* msg = NULL;
 
 
     if ((ret = srs_rtmp_create_msg(type, timestamp, data, size, stream_id, &msg)) != ERROR_SUCCESS) {
     if ((ret = srs_rtmp_create_msg(type, timestamp, data, size, stream_id, &msg)) != ERROR_SUCCESS) {
+        srs_error("mpegts: create shared ptr msg failed. ret=%d", ret);
         return ret;
         return ret;
     }
     }
-
     srs_assert(msg);
     srs_assert(msg);
 
 
+    // push msg to queue.
+    if ((ret = queue->push(msg)) != ERROR_SUCCESS) {
+        srs_error("mpegts: push msg to queue failed. ret=%d", ret);
+        return ret;
+    }
+
+    // for all ready msg, dequeue and send out.
+    for (;;) {
+        if ((msg = queue->dequeue()) == NULL) {
+            break;
+        }
+    
         // send out encoded msg.
         // send out encoded msg.
         if ((ret = client->send_and_free_message(msg, stream_id)) != ERROR_SUCCESS) {
         if ((ret = client->send_and_free_message(msg, stream_id)) != ERROR_SUCCESS) {
             return ret;
             return ret;
         }
         }
+    }
     
     
     return ret;
     return ret;
 }
 }

@@ -34,6 +34,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 
 class sockaddr_in;
 class sockaddr_in;
 #include <string>
 #include <string>
+#include <map>
 
 
 class SrsStream;
 class SrsStream;
 class SrsTsContext;
 class SrsTsContext;
@@ -43,6 +44,7 @@ class SrsRtmpClient;
 class SrsStSocket;
 class SrsStSocket;
 class SrsRequest;
 class SrsRequest;
 class SrsRawH264Stream;
 class SrsRawH264Stream;
+class SrsSharedPtrMessage;
 
 
 #include <srs_app_st.hpp>
 #include <srs_app_st.hpp>
 #include <srs_kernel_ts.hpp>
 #include <srs_kernel_ts.hpp>
@@ -69,6 +71,26 @@ public:
 };
 };
 
 
 /**
 /**
+* the queue for mpegts over udp to send packets.
+* for the aac in mpegts contains many flv packets in a pes packet,
+* we must recalc the timestamp.
+*/
+class SrsMpegtsQueue
+{
+private:
+    // key: dts, value: msg.
+    std::map<int64_t, SrsSharedPtrMessage*> msgs;
+    int nb_audios;
+    int nb_videos;
+public:
+    SrsMpegtsQueue();
+    virtual ~SrsMpegtsQueue();
+public:
+    virtual int push(SrsSharedPtrMessage* msg);
+    virtual SrsSharedPtrMessage* dequeue();
+};
+
+/**
 * the mpegts over udp stream caster.
 * the mpegts over udp stream caster.
 */
 */
 class SrsMpegtsOverUdp : virtual public ISrsTsHandler
 class SrsMpegtsOverUdp : virtual public ISrsTsHandler
@@ -92,6 +114,7 @@ private:
     std::string h264_pps;
     std::string h264_pps;
     bool h264_pps_changed;
     bool h264_pps_changed;
     bool h264_sps_pps_sent;
     bool h264_sps_pps_sent;
+    SrsMpegtsQueue* queue;
 public:
 public:
     SrsMpegtsOverUdp(SrsConfDirective* c);
     SrsMpegtsOverUdp(SrsConfDirective* c);
     virtual ~SrsMpegtsOverUdp();
     virtual ~SrsMpegtsOverUdp();

@@ -40,7 +40,7 @@ using namespace std;
 #include <srs_kernel_stream.hpp>
 #include <srs_kernel_stream.hpp>
 #include <srs_app_edge.hpp>
 #include <srs_app_edge.hpp>
 #include <srs_kernel_utility.hpp>
 #include <srs_kernel_utility.hpp>
-#include <srs_kernel_avc.hpp>
+#include <srs_kernel_codec.hpp>
 #include <srs_rtmp_msg_array.hpp>
 #include <srs_rtmp_msg_array.hpp>
 
 
 #define CONST_MAX_JITTER_MS         500
 #define CONST_MAX_JITTER_MS         500

-/*
-The MIT License (MIT)
-
-Copyright (c) 2013-2015 winlin
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
-
-#include <srs_kernel_avc.hpp>
-
-#include <srs_kernel_error.hpp>
-#include <srs_kernel_log.hpp>
-#include <srs_kernel_stream.hpp>
-#include <srs_kernel_utility.hpp>
-#include <srs_kernel_buffer.hpp>
-#include <srs_kernel_file.hpp>
-
-using namespace std;
-
-SrsCodecSampleUnit::SrsCodecSampleUnit()
-{
-    size = 0;
-    bytes = NULL;
-}
-
-SrsCodecSampleUnit::~SrsCodecSampleUnit()
-{
-}
-
-SrsCodecSample::SrsCodecSample()
-{
-    clear();
-}
-
-SrsCodecSample::~SrsCodecSample()
-{
-}
-
-void SrsCodecSample::clear()
-{
-    is_video = false;
-    nb_sample_units = 0;
-
-    cts = 0;
-    frame_type = SrsCodecVideoAVCFrameReserved;
-    avc_packet_type = SrsCodecVideoAVCTypeReserved;
-    
-    acodec = SrsCodecAudioReserved1;
-    sound_rate = SrsCodecAudioSampleRateReserved;
-    sound_size = SrsCodecAudioSampleSizeReserved;
-    sound_type = SrsCodecAudioSoundTypeReserved;
-    aac_packet_type = SrsCodecAudioTypeReserved;
-}
-
-int SrsCodecSample::add_sample_unit(char* bytes, int size)
-{
-    int ret = ERROR_SUCCESS;
-    
-    if (nb_sample_units >= __SRS_SRS_MAX_CODEC_SAMPLE) {
-        ret = ERROR_HLS_DECODE_ERROR;
-        srs_error("hls decode samples error, "
-            "exceed the max count: %d, ret=%d", __SRS_SRS_MAX_CODEC_SAMPLE, ret);
-        return ret;
-    }
-    
-    SrsCodecSampleUnit* sample_unit = &sample_units[nb_sample_units++];
-    sample_unit->bytes = bytes;
-    sample_unit->size = size;
-    
-    return ret;
-}
-
-SrsAvcAacCodec::SrsAvcAacCodec()
-{
-    width                       = 0;
-    height                      = 0;
-    duration                    = 0;
-    NAL_unit_length             = 0;
-    frame_rate                  = 0;
-
-    video_data_rate             = 0;
-    video_codec_id              = 0;
-
-    audio_data_rate             = 0;
-    audio_codec_id              = 0;
-
-    avc_profile                 = 0;
-    avc_level                   = 0;
-    aac_profile                 = 0;
-    aac_sample_rate             = __SRS_AAC_SAMPLE_RATE_UNSET; // sample rate ignored
-    aac_channels                = 0;
-    avc_extra_size              = 0;
-    avc_extra_data              = NULL;
-    aac_extra_size              = 0;
-    aac_extra_data              = NULL;
-
-    sequenceParameterSetLength  = 0;
-    sequenceParameterSetNALUnit = NULL;
-    pictureParameterSetLength   = 0;
-    pictureParameterSetNALUnit  = NULL;
-
-    stream = new SrsStream();
-}
-
-SrsAvcAacCodec::~SrsAvcAacCodec()
-{
-    srs_freep(avc_extra_data);
-    srs_freep(aac_extra_data);
-
-    srs_freep(stream);
-    srs_freep(sequenceParameterSetNALUnit);
-    srs_freep(pictureParameterSetNALUnit);
-}
-
-int SrsAvcAacCodec::audio_aac_demux(char* data, int size, SrsCodecSample* sample)
-{
-    int ret = ERROR_SUCCESS;
-    
-    sample->is_video = false;
-    
-    if (!data || size <= 0) {
-        srs_trace("no audio present, ignore it.");
-        return ret;
-    }
-    
-    if ((ret = stream->initialize(data, size)) != ERROR_SUCCESS) {
-        return ret;
-    }
-
-    // audio decode
-    if (!stream->require(1)) {
-        ret = ERROR_HLS_DECODE_ERROR;
-        srs_error("audio codec decode sound_format failed. ret=%d", ret);
-        return ret;
-    }
-    
-    // @see: E.4.2 Audio Tags, video_file_format_spec_v10_1.pdf, page 76
-    int8_t sound_format = stream->read_1bytes();
-    
-    int8_t sound_type = sound_format & 0x01;
-    int8_t sound_size = (sound_format >> 1) & 0x01;
-    int8_t sound_rate = (sound_format >> 2) & 0x03;
-    sound_format = (sound_format >> 4) & 0x0f;
-    
-    audio_codec_id = sound_format;
-    sample->acodec = (SrsCodecAudio)audio_codec_id;
-
-    sample->sound_type = (SrsCodecAudioSoundType)sound_type;
-    sample->sound_rate = (SrsCodecAudioSampleRate)sound_rate;
-    sample->sound_size = (SrsCodecAudioSampleSize)sound_size;
-
-    // we support h.264+mp3 for hls.
-    if (audio_codec_id == SrsCodecAudioMP3) {
-        return ERROR_HLS_TRY_MP3;
-    }
-    
-    // only support aac
-    if (audio_codec_id != SrsCodecAudioAAC) {
-        ret = ERROR_HLS_DECODE_ERROR;
-        srs_error("audio codec only support mp3/aac codec. actual=%d, ret=%d", audio_codec_id, ret);
-        return ret;
-    }
-
-    if (!stream->require(1)) {
-        ret = ERROR_HLS_DECODE_ERROR;
-        srs_error("audio codec decode aac_packet_type failed. ret=%d", ret);
-        return ret;
-    }
-    
-    int8_t aac_packet_type = stream->read_1bytes();
-    sample->aac_packet_type = (SrsCodecAudioType)aac_packet_type;
-    
-    if (aac_packet_type == SrsCodecAudioTypeSequenceHeader) {
-        // AudioSpecificConfig
-        // 1.6.2.1 AudioSpecificConfig, in aac-mp4a-format-ISO_IEC_14496-3+2001.pdf, page 33.
-        aac_extra_size = stream->size() - stream->pos();
-        if (aac_extra_size > 0) {
-            srs_freep(aac_extra_data);
-            aac_extra_data = new char[aac_extra_size];
-            memcpy(aac_extra_data, stream->data() + stream->pos(), aac_extra_size);
-        }
-        
-        // only need to decode the first 2bytes:
-        // audioObjectType, aac_profile, 5bits.
-        // samplingFrequencyIndex, aac_sample_rate, 4bits.
-        // channelConfiguration, aac_channels, 4bits
-        if (!stream->require(2)) {
-            ret = ERROR_HLS_DECODE_ERROR;
-            srs_error("audio codec decode aac sequence header failed. ret=%d", ret);
-            return ret;
-        }
-        aac_profile = stream->read_1bytes();
-        aac_sample_rate = stream->read_1bytes();
-        
-        aac_channels = (aac_sample_rate >> 3) & 0x0f;
-        aac_sample_rate = ((aac_profile << 1) & 0x0e) | ((aac_sample_rate >> 7) & 0x01);
-        aac_profile = (aac_profile >> 3) & 0x1f;
-        
-        if (aac_profile == 0 || aac_profile == 0x1f) {
-            ret = ERROR_HLS_DECODE_ERROR;
-            srs_error("audio codec decode aac sequence header failed, "
-                "adts object=%d invalid. ret=%d", aac_profile, ret);
-            return ret;
-        }
-        
-        // TODO: FIXME: to support aac he/he-v2, see: ngx_rtmp_codec_parse_aac_header
-        // @see: https://github.com/winlinvip/nginx-rtmp-module/commit/3a5f9eea78fc8d11e8be922aea9ac349b9dcbfc2
-        // 
-        // donot force to LC, @see: https://github.com/winlinvip/simple-rtmp-server/issues/81
-        // the source will print the sequence header info.
-        //if (aac_profile > 3) {
-            // Mark all extended profiles as LC
-            // to make Android as happy as possible.
-            // @see: ngx_rtmp_hls_parse_aac_header
-            //aac_profile = 1;
-        //}
-    } else if (aac_packet_type == SrsCodecAudioTypeRawData) {
-        // ensure the sequence header demuxed
-        if (aac_extra_size <= 0 || !aac_extra_data) {
-            ret = ERROR_HLS_DECODE_ERROR;
-            srs_error("audio codec decode aac failed, sequence header not found. ret=%d", ret);
-            return ret;
-        }
-        
-        // Raw AAC frame data in UI8 []
-        // 6.3 Raw Data, aac-iso-13818-7.pdf, page 28
-        if ((ret = sample->add_sample_unit(stream->data() + stream->pos(), stream->size() - stream->pos())) != ERROR_SUCCESS) {
-            srs_error("audio codec add sample failed. ret=%d", ret);
-            return ret;
-        }
-    } else {
-        // ignored.
-    }
-    
-    // reset the sample rate by sequence header
-    if (aac_sample_rate != __SRS_AAC_SAMPLE_RATE_UNSET) {
-        static int aac_sample_rates[] = {
-            96000, 88200, 64000, 48000,
-            44100, 32000, 24000, 22050,
-            16000, 12000, 11025,  8000,
-            7350,     0,     0,    0
-        };
-        switch (aac_sample_rates[aac_sample_rate]) {
-            case 11025:
-                sample->sound_rate = SrsCodecAudioSampleRate11025;
-                break;
-            case 22050:
-                sample->sound_rate = SrsCodecAudioSampleRate22050;
-                break;
-            case 44100:
-                sample->sound_rate = SrsCodecAudioSampleRate44100;
-                break;
-            default:
-                break;
-        };
-    }
-    
-    srs_info("audio decoded, type=%d, codec=%d, asize=%d, rate=%d, format=%d, size=%d", 
-        sound_type, audio_codec_id, sound_size, sound_rate, sound_format, size);
-    
-    return ret;
-}
-
-int SrsAvcAacCodec::audio_mp3_demux(char* data, int size, SrsCodecSample* sample)
-{
-    int ret = ERROR_SUCCESS;
-
-    // we always decode aac then mp3.
-    srs_assert(sample->acodec == SrsCodecAudioMP3);
-    
-    // @see: E.4.2 Audio Tags, video_file_format_spec_v10_1.pdf, page 76
-    if (!data || size <= 1) {
-        srs_trace("no mp3 audio present, ignore it.");
-        return ret;
-    }
-
-    // mp3 payload.
-    if ((ret = sample->add_sample_unit(data + 1, size - 1)) != ERROR_SUCCESS) {
-        srs_error("audio codec add mp3 sample failed. ret=%d", ret);
-        return ret;
-    }
-    
-    srs_info("audio decoded, type=%d, codec=%d, asize=%d, rate=%d, format=%d, size=%d", 
-        sample->sound_type, audio_codec_id, sample->sound_size, sample->sound_rate, sample->acodec, size);
-    
-    return ret;
-}
-
-int SrsAvcAacCodec::video_avc_demux(char* data, int size, SrsCodecSample* sample)
-{
-    int ret = ERROR_SUCCESS;
-    
-    sample->is_video = true;
-    
-    if (!data || size <= 0) {
-        srs_trace("no video present, ignore it.");
-        return ret;
-    }
-    
-    if ((ret = stream->initialize(data, size)) != ERROR_SUCCESS) {
-        return ret;
-    }
-
-    // video decode
-    if (!stream->require(1)) {
-        ret = ERROR_HLS_DECODE_ERROR;
-        srs_error("video codec decode frame_type failed. ret=%d", ret);
-        return ret;
-    }
-    
-    // @see: E.4.3 Video Tags, video_file_format_spec_v10_1.pdf, page 78
-    int8_t frame_type = stream->read_1bytes();
-    int8_t codec_id = frame_type & 0x0f;
-    frame_type = (frame_type >> 4) & 0x0f;
-    
-    sample->frame_type = (SrsCodecVideoAVCFrame)frame_type;
-    
-    // ignore info frame without error,
-    // @see https://github.com/winlinvip/simple-rtmp-server/issues/288#issuecomment-69863909
-    if (sample->frame_type == SrsCodecVideoAVCFrameVideoInfoFrame) {
-        srs_warn("video codec igone the info frame, ret=%d", ret);
-        return ret;
-    }
-    
-    // only support h.264/avc
-    if (codec_id != SrsCodecVideoAVC) {
-        ret = ERROR_HLS_DECODE_ERROR;
-        srs_error("video codec only support video h.264/avc codec. actual=%d, ret=%d", codec_id, ret);
-        return ret;
-    }
-    video_codec_id = codec_id;
-    
-    if (!stream->require(4)) {
-        ret = ERROR_HLS_DECODE_ERROR;
-        srs_error("video codec decode avc_packet_type failed. ret=%d", ret);
-        return ret;
-    }
-    int8_t avc_packet_type = stream->read_1bytes();
-    int32_t composition_time = stream->read_3bytes();
-    
-    // pts = dts + cts.
-    sample->cts = composition_time;
-    sample->avc_packet_type = (SrsCodecVideoAVCType)avc_packet_type;
-    
-    if (avc_packet_type == SrsCodecVideoAVCTypeSequenceHeader) {
-        if ((ret = avc_demux_sps_pps(stream)) != ERROR_SUCCESS) {
-            return ret;
-        }
-    } else if (avc_packet_type == SrsCodecVideoAVCTypeNALU){
-        // ensure the sequence header demuxed
-        if (avc_extra_size <= 0 || !avc_extra_data) {
-            ret = ERROR_HLS_DECODE_ERROR;
-            srs_error("avc decode failed, sequence header not found. ret=%d", ret);
-            return ret;
-        }
-        
-        // One or more NALUs (Full frames are required)
-        // try  "AnnexB" from H.264-AVC-ISO_IEC_14496-10.pdf, page 211.
-        if ((ret = avc_demux_annexb_format(stream, sample)) != ERROR_SUCCESS) {
-            // stop try when system error.
-            if (ret != ERROR_HLS_AVC_TRY_OTHERS) {
-                srs_error("avc demux for annexb failed. ret=%d", ret);
-                return ret;
-            }
-            
-            // try "ISO Base Media File Format" from H.264-AVC-ISO_IEC_14496-15.pdf, page 20
-            if ((ret = avc_demux_ibmf_format(stream, sample)) != ERROR_SUCCESS) {
-                return ret;
-            }
-        }
-    } else {
-        // ignored.
-    }
-    
-    srs_info("video decoded, type=%d, codec=%d, avc=%d, time=%d, size=%d", 
-        frame_type, video_codec_id, avc_packet_type, composition_time, size);
-    
-    return ret;
-}
-
-int SrsAvcAacCodec::avc_demux_sps_pps(SrsStream* stream)
-{
-    int ret = ERROR_SUCCESS;
-    
-    // AVCDecoderConfigurationRecord
-    // 5.2.4.1.1 Syntax, H.264-AVC-ISO_IEC_14496-15.pdf, page 16
-    avc_extra_size = stream->size() - stream->pos();
-    if (avc_extra_size > 0) {
-        srs_freep(avc_extra_data);
-        avc_extra_data = new char[avc_extra_size];
-        memcpy(avc_extra_data, stream->data() + stream->pos(), avc_extra_size);
-    }
-    
-    if (!stream->require(6)) {
-        ret = ERROR_HLS_DECODE_ERROR;
-        srs_error("avc decode sequenc header failed. ret=%d", ret);
-        return ret;
-    }
-    //int8_t configurationVersion = stream->read_1bytes();
-    stream->read_1bytes();
-    //int8_t AVCProfileIndication = stream->read_1bytes();
-    avc_profile = stream->read_1bytes();
-    //int8_t profile_compatibility = stream->read_1bytes();
-    stream->read_1bytes();
-    //int8_t AVCLevelIndication = stream->read_1bytes();
-    avc_level = stream->read_1bytes();
-    
-    // parse the NALU size.
-    int8_t lengthSizeMinusOne = stream->read_1bytes();
-    lengthSizeMinusOne &= 0x03;
-    NAL_unit_length = lengthSizeMinusOne;
-    
-    // 5.3.4.2.1 Syntax, H.264-AVC-ISO_IEC_14496-15.pdf, page 16
-    // 5.2.4.1 AVC decoder configuration record
-    // 5.2.4.1.2 Semantics
-    // The value of this field shall be one of 0, 1, or 3 corresponding to a
-    // length encoded with 1, 2, or 4 bytes, respectively.
-    if (NAL_unit_length == 2) {
-        ret = ERROR_HLS_DECODE_ERROR;
-        srs_error("sps lengthSizeMinusOne should never be 2. ret=%d", ret);
-        return ret;
-    }
-    
-    // 1 sps
-    if (!stream->require(1)) {
-        ret = ERROR_HLS_DECODE_ERROR;
-        srs_error("avc decode sequenc header sps failed. ret=%d", ret);
-        return ret;
-    }
-    int8_t numOfSequenceParameterSets = stream->read_1bytes();
-    numOfSequenceParameterSets &= 0x1f;
-    if (numOfSequenceParameterSets != 1) {
-        ret = ERROR_HLS_DECODE_ERROR;
-        srs_error("avc decode sequenc header sps failed. ret=%d", ret);
-        return ret;
-    }
-    if (!stream->require(2)) {
-        ret = ERROR_HLS_DECODE_ERROR;
-        srs_error("avc decode sequenc header sps size failed. ret=%d", ret);
-        return ret;
-    }
-    sequenceParameterSetLength = stream->read_2bytes();
-    if (!stream->require(sequenceParameterSetLength)) {
-        ret = ERROR_HLS_DECODE_ERROR;
-        srs_error("avc decode sequenc header sps data failed. ret=%d", ret);
-        return ret;
-    }
-    if (sequenceParameterSetLength > 0) {
-        srs_freep(sequenceParameterSetNALUnit);
-        sequenceParameterSetNALUnit = new char[sequenceParameterSetLength];
-        memcpy(sequenceParameterSetNALUnit, stream->data() + stream->pos(), sequenceParameterSetLength);
-        stream->skip(sequenceParameterSetLength);
-    }
-    // 1 pps
-    if (!stream->require(1)) {
-        ret = ERROR_HLS_DECODE_ERROR;
-        srs_error("avc decode sequenc header pps failed. ret=%d", ret);
-        return ret;
-    }
-    int8_t numOfPictureParameterSets = stream->read_1bytes();
-    numOfPictureParameterSets &= 0x1f;
-    if (numOfPictureParameterSets != 1) {
-        ret = ERROR_HLS_DECODE_ERROR;
-        srs_error("avc decode sequenc header pps failed. ret=%d", ret);
-        return ret;
-    }
-    if (!stream->require(2)) {
-        ret = ERROR_HLS_DECODE_ERROR;
-        srs_error("avc decode sequenc header pps size failed. ret=%d", ret);
-        return ret;
-    }
-    pictureParameterSetLength = stream->read_2bytes();
-    if (!stream->require(pictureParameterSetLength)) {
-        ret = ERROR_HLS_DECODE_ERROR;
-        srs_error("avc decode sequenc header pps data failed. ret=%d", ret);
-        return ret;
-    }
-    if (pictureParameterSetLength > 0) {
-        srs_freep(pictureParameterSetNALUnit);
-        pictureParameterSetNALUnit = new char[pictureParameterSetLength];
-        memcpy(pictureParameterSetNALUnit, stream->data() + stream->pos(), pictureParameterSetLength);
-        stream->skip(pictureParameterSetLength);
-    }
-    
-    return ret;
-}
-
-int SrsAvcAacCodec::avc_demux_annexb_format(SrsStream* stream, SrsCodecSample* sample)
-{
-    int ret = ERROR_SUCCESS;
-    
-    // not annexb, try others
-    if (!srs_avc_startswith_annexb(stream, NULL)) {
-        return ERROR_HLS_AVC_TRY_OTHERS;
-    }
-    
-    // AnnexB
-    // B.1.1 Byte stream NAL unit syntax,
-    // H.264-AVC-ISO_IEC_14496-10.pdf, page 211.
-    while (!stream->empty()) {
-        // find start code
-        int nb_start_code = 0;
-        if (!srs_avc_startswith_annexb(stream, &nb_start_code)) {
-            return ret;
-        }
-
-        // skip the start code.
-        if (nb_start_code > 0) {
-            stream->skip(nb_start_code);
-        }
-        
-        // the NALU start bytes.
-        char* p = stream->data() + stream->pos();
-        
-        // get the last matched NALU
-        while (!stream->empty()) {
-            if (srs_avc_startswith_annexb(stream, NULL)) {
-                break;
-            }
-            
-            stream->skip(1);
-        }
-        
-        char* pp = stream->data() + stream->pos();
-        
-        // skip the empty.
-        if (pp - p <= 0) {
-            continue;
-        }
-        
-        // got the NALU.
-        if ((ret = sample->add_sample_unit(p, pp - p)) != ERROR_SUCCESS) {
-            srs_error("annexb add video sample failed. ret=%d", ret);
-            return ret;
-        }
-    }
-    
-    return ret;
-}
-
-int SrsAvcAacCodec::avc_demux_ibmf_format(SrsStream* stream, SrsCodecSample* sample)
-{
-    int ret = ERROR_SUCCESS;
-    
-    int PictureLength = stream->size() - stream->pos();
-    
-    // 5.3.4.2.1 Syntax, H.264-AVC-ISO_IEC_14496-15.pdf, page 16
-    // 5.2.4.1 AVC decoder configuration record
-    // 5.2.4.1.2 Semantics
-    // The value of this field shall be one of 0, 1, or 3 corresponding to a
-    // length encoded with 1, 2, or 4 bytes, respectively.
-    srs_assert(NAL_unit_length != 2);
-    
-    // 5.3.4.2.1 Syntax, H.264-AVC-ISO_IEC_14496-15.pdf, page 20
-    for (int i = 0; i < PictureLength;) {
-        // unsigned int((NAL_unit_length+1)*8) NALUnitLength;
-        if (!stream->require(NAL_unit_length + 1)) {
-            ret = ERROR_HLS_DECODE_ERROR;
-            srs_error("avc decode NALU size failed. ret=%d", ret);
-            return ret;
-        }
-        int32_t NALUnitLength = 0;
-        if (NAL_unit_length == 3) {
-            NALUnitLength = stream->read_4bytes();
-        } else if (NAL_unit_length == 1) {
-            NALUnitLength = stream->read_2bytes();
-        } else {
-            NALUnitLength = stream->read_1bytes();
-        }
-        
-        // maybe stream is invalid format.
-        // see: https://github.com/winlinvip/simple-rtmp-server/issues/183
-        if (NALUnitLength < 0) {
-            ret = ERROR_HLS_DECODE_ERROR;
-            srs_error("maybe stream is AnnexB format. ret=%d", ret);
-            return ret;
-        }
-        
-        // NALUnit
-        if (!stream->require(NALUnitLength)) {
-            ret = ERROR_HLS_DECODE_ERROR;
-            srs_error("avc decode NALU data failed. ret=%d", ret);
-            return ret;
-        }
-        // 7.3.1 NAL unit syntax, H.264-AVC-ISO_IEC_14496-10.pdf, page 44.
-        if ((ret = sample->add_sample_unit(stream->data() + stream->pos(), NALUnitLength)) != ERROR_SUCCESS) {
-            srs_error("avc add video sample failed. ret=%d", ret);
-            return ret;
-        }
-        stream->skip(NALUnitLength);
-        
-        i += NAL_unit_length + 1 + NALUnitLength;
-    }
-    
-    return ret;
-}
-

-/*
-The MIT License (MIT)
-
-Copyright (c) 2013-2015 winlin
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
-
-#ifndef SRS_KERNEL_AVC_HPP
-#define SRS_KERNEL_AVC_HPP
-
-/*
-#include <srs_kernel_avc.hpp>
-*/
-
-#include <srs_core.hpp>
-
-#include <string>
-
-#include <srs_kernel_codec.hpp>
-
-class SrsStream;
-class SrsMpegtsFrame;
-class SrsSimpleBuffer;
-class SrsAvcAacCodec;
-class SrsCodecSample;
-class SrsFileWriter;
-
-/**
-* the public data, event HLS disable, others can use it.
-*/
-/**
-* the flv sample rate map
-*/
-extern int flv_sample_rates[];
-
-/**
-* the aac sample rate map
-*/
-extern int aac_sample_rates[];
-
-#define __SRS_SRS_MAX_CODEC_SAMPLE 128
-#define __SRS_AAC_SAMPLE_RATE_UNSET 15
-
-// in ms, for HLS aac flush the audio
-#define SRS_CONF_DEFAULT_AAC_DELAY 100
-
-// max PES packets size to flush the video.
-#define SRS_AUTO_HLS_AUDIO_CACHE_SIZE 1024 * 1024
-
-/**
-* the FLV/RTMP supported audio sample size.
-* Size of each audio sample. This parameter only pertains to
-* uncompressed formats. Compressed formats always decode
-* to 16 bits internally.
-* 0 = 8-bit samples
-* 1 = 16-bit samples
-*/
-enum SrsCodecAudioSampleSize
-{
-    // set to the max value to reserved, for array map.
-    SrsCodecAudioSampleSizeReserved                 = 2,
-    
-    SrsCodecAudioSampleSize8bit                     = 0,
-    SrsCodecAudioSampleSize16bit                    = 1,
-};
-
-/**
-* the FLV/RTMP supported audio sound type/channel.
-* Mono or stereo sound
-* 0 = Mono sound
-* 1 = Stereo sound
-*/
-enum SrsCodecAudioSoundType
-{
-    // set to the max value to reserved, for array map.
-    SrsCodecAudioSoundTypeReserved                  = 2, 
-    
-    SrsCodecAudioSoundTypeMono                      = 0,
-    SrsCodecAudioSoundTypeStereo                    = 1,
-};
-
-/**
-* the codec sample unit.
-* for h.264 video packet, a NALU is a sample unit.
-* for aac raw audio packet, a NALU is the entire aac raw data.
-* for sequence header, it's not a sample unit.
-*/
-class SrsCodecSampleUnit
-{
-public:
-    /**
-    * the sample bytes is directly ptr to packet bytes,
-    * user should never use it when packet destroyed.
-    */
-    int size;
-    char* bytes;
-public:
-    SrsCodecSampleUnit();
-    virtual ~SrsCodecSampleUnit();
-};
-
-/**
-* the samples in the flv audio/video packet.
-* the sample used to analysis a video/audio packet,
-* split the h.264 NALUs to buffers, or aac raw data to a buffer,
-* and decode the video/audio specified infos.
-* 
-* the sample unit:
-*       a video packet codec in h.264 contains many NALUs, each is a sample unit.
-*       a audio packet codec in aac is a sample unit.
-* @remark, the video/audio sequence header is not sample unit,
-*       all sequence header stores as extra data, 
-*       @see SrsAvcAacCodec.avc_extra_data and SrsAvcAacCodec.aac_extra_data
-* @remark, user must clear all samples before decode a new video/audio packet.
-*/
-class SrsCodecSample
-{
-public:
-    /**
-    * each audio/video raw data packet will dumps to one or multiple buffers,
-    * the buffers will write to hls and clear to reset.
-    * generally, aac audio packet corresponding to one buffer,
-    * where avc/h264 video packet may contains multiple buffer.
-    */
-    int nb_sample_units;
-    SrsCodecSampleUnit sample_units[__SRS_SRS_MAX_CODEC_SAMPLE];
-public:
-    /**
-    * whether the sample is video sample which demux from video packet.
-    */
-    bool is_video;
-    /**
-    * CompositionTime, video_file_format_spec_v10_1.pdf, page 78.
-    * cts = pts - dts, where dts = flvheader->timestamp.
-    */
-    int32_t cts;
-public:
-    // video specified
-    SrsCodecVideoAVCFrame frame_type;
-    SrsCodecVideoAVCType avc_packet_type;
-public:
-    // audio specified
-    SrsCodecAudio acodec;
-    // audio aac specified.
-    SrsCodecAudioSampleRate sound_rate;
-    SrsCodecAudioSampleSize sound_size;
-    SrsCodecAudioSoundType sound_type;
-    SrsCodecAudioType aac_packet_type;
-public:
-    SrsCodecSample();
-    virtual ~SrsCodecSample();
-public:
-    /**
-    * clear all samples.
-    * the sample units never copy the bytes, it directly use the ptr,
-    * so when video/audio packet is destroyed, the sample must be clear.
-    * in a word, user must clear sample before demux it.
-    * @remark demux sample use SrsAvcAacCodec.audio_aac_demux or video_avc_demux.
-    */
-    void clear();
-    /**
-    * add the a sample unit, it's a h.264 NALU or aac raw data.
-    * the sample unit directly use the ptr of packet bytes,
-    * so user must never use sample unit when packet is destroyed.
-    * in a word, user must clear sample before demux it.
-    */
-    int add_sample_unit(char* bytes, int size);
-};
-
-/**
-* the h264/avc and aac codec, for media stream.
-*
-* to demux the FLV/RTMP video/audio packet to sample,
-* add each NALUs of h.264 as a sample unit to sample,
-* while the entire aac raw data as a sample unit.
-*
-* for sequence header,
-* demux it and save it in the avc_extra_data and aac_extra_data,
-* 
-* for the codec info, such as audio sample rate,
-* decode from FLV/RTMP header, then use codec info in sequence 
-* header to override it.
-*/
-class SrsAvcAacCodec
-{
-private:
-    SrsStream* stream;
-public:
-    /**
-    * metadata specified
-    */
-    int             duration;
-    int             width;
-    int             height;
-    int             frame_rate;
-    // @see: SrsCodecVideo
-    int             video_codec_id;
-    int             video_data_rate; // in bps
-    // @see: SrsCod ecAudioType
-    int             audio_codec_id;
-    int             audio_data_rate; // in bps
-public:
-    /**
-    * video specified
-    */
-    // profile_idc, H.264-AVC-ISO_IEC_14496-10.pdf, page 45.
-    u_int8_t        avc_profile; 
-    // level_idc, H.264-AVC-ISO_IEC_14496-10.pdf, page 45.
-    u_int8_t        avc_level; 
-    // lengthSizeMinusOne, H.264-AVC-ISO_IEC_14496-15.pdf, page 16
-    int8_t          NAL_unit_length;
-    u_int16_t       sequenceParameterSetLength;
-    char*           sequenceParameterSetNALUnit;
-    u_int16_t       pictureParameterSetLength;
-    char*           pictureParameterSetNALUnit;
-public:
-    /**
-    * audio specified
-    * audioObjectType, in 1.6.2.1 AudioSpecificConfig, page 33,
-    * 1.5.1.1 Audio object type definition, page 23,
-    *           in aac-mp4a-format-ISO_IEC_14496-3+2001.pdf.
-    */
-    u_int8_t        aac_profile; 
-    /**
-    * samplingFrequencyIndex
-    */
-    u_int8_t        aac_sample_rate;
-    /**
-    * channelConfiguration
-    */
-    u_int8_t        aac_channels;
-public:
-    /**
-    * the avc extra data, the AVC sequence header,
-    * without the flv codec header,
-    * @see: ffmpeg, AVCodecContext::extradata
-    */
-    int             avc_extra_size;
-    char*           avc_extra_data;
-    /**
-    * the aac extra data, the AAC sequence header,
-    * without the flv codec header,
-    * @see: ffmpeg, AVCodecContext::extradata
-    */
-    int             aac_extra_size;
-    char*           aac_extra_data;
-public:
-    SrsAvcAacCodec();
-    virtual ~SrsAvcAacCodec();
-// the following function used for hls to build the sample and codec.
-public:
-    /**
-    * demux the audio packet in aac codec.
-    * the packet mux in FLV/RTMP format defined in flv specification.
-    * demux the audio speicified data(sound_format, sound_size, ...) to sample.
-    * demux the aac specified data(aac_profile, ...) to codec from sequence header.
-    * demux the aac raw to sample units.
-    */
-    virtual int audio_aac_demux(char* data, int size, SrsCodecSample* sample);
-    virtual int audio_mp3_demux(char* data, int size, SrsCodecSample* sample);
-    /**
-    * demux the video packet in h.264 codec.
-    * the packet mux in FLV/RTMP format defined in flv specification.
-    * demux the video specified data(frame_type, codec_id, ...) to sample.
-    * demux the h.264 sepcified data(avc_profile, ...) to codec from sequence header.
-    * demux the h.264 NALUs to sampe units.
-    */
-    virtual int video_avc_demux(char* data, int size, SrsCodecSample* sample);
-private:
-    /**
-    * when avc packet type is SrsCodecVideoAVCTypeSequenceHeader,
-    * decode the sps and pps.
-    */
-    virtual int avc_demux_sps_pps(SrsStream* stream);
-    /**
-    * demux the avc NALU in "AnnexB" 
-    * from H.264-AVC-ISO_IEC_14496-10.pdf, page 211.
-    */
-    virtual int avc_demux_annexb_format(SrsStream* stream, SrsCodecSample* sample);
-    /**
-    * demux the avc NALU in "ISO Base Media File Format" 
-    * from H.264-AVC-ISO_IEC_14496-15.pdf, page 20
-    */
-    virtual int avc_demux_ibmf_format(SrsStream* stream, SrsCodecSample* sample);
-};
-
-#endif
-

@@ -25,6 +25,12 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 
 #include <string.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdlib.h>
+using namespace std;
+
+#include <srs_kernel_error.hpp>
+#include <srs_kernel_log.hpp>
+#include <srs_kernel_stream.hpp>
+#include <srs_kernel_utility.hpp>
 
 
 SrsFlvCodec::SrsFlvCodec()
 SrsFlvCodec::SrsFlvCodec()
 {
 {
@@ -111,3 +117,581 @@ bool SrsFlvCodec::audio_is_aac(char* data, int size)
     return sound_format == SrsCodecAudioAAC;
     return sound_format == SrsCodecAudioAAC;
 }
 }
 
 
+SrsCodecSampleUnit::SrsCodecSampleUnit()
+{
+    size = 0;
+    bytes = NULL;
+}
+
+SrsCodecSampleUnit::~SrsCodecSampleUnit()
+{
+}
+
+SrsCodecSample::SrsCodecSample()
+{
+    clear();
+}
+
+SrsCodecSample::~SrsCodecSample()
+{
+}
+
+void SrsCodecSample::clear()
+{
+    is_video = false;
+    nb_sample_units = 0;
+
+    cts = 0;
+    frame_type = SrsCodecVideoAVCFrameReserved;
+    avc_packet_type = SrsCodecVideoAVCTypeReserved;
+    
+    acodec = SrsCodecAudioReserved1;
+    sound_rate = SrsCodecAudioSampleRateReserved;
+    sound_size = SrsCodecAudioSampleSizeReserved;
+    sound_type = SrsCodecAudioSoundTypeReserved;
+    aac_packet_type = SrsCodecAudioTypeReserved;
+}
+
+int SrsCodecSample::add_sample_unit(char* bytes, int size)
+{
+    int ret = ERROR_SUCCESS;
+    
+    if (nb_sample_units >= __SRS_SRS_MAX_CODEC_SAMPLE) {
+        ret = ERROR_HLS_DECODE_ERROR;
+        srs_error("hls decode samples error, "
+            "exceed the max count: %d, ret=%d", __SRS_SRS_MAX_CODEC_SAMPLE, ret);
+        return ret;
+    }
+    
+    SrsCodecSampleUnit* sample_unit = &sample_units[nb_sample_units++];
+    sample_unit->bytes = bytes;
+    sample_unit->size = size;
+    
+    return ret;
+}
+
+SrsAvcAacCodec::SrsAvcAacCodec()
+{
+    width                       = 0;
+    height                      = 0;
+    duration                    = 0;
+    NAL_unit_length             = 0;
+    frame_rate                  = 0;
+
+    video_data_rate             = 0;
+    video_codec_id              = 0;
+
+    audio_data_rate             = 0;
+    audio_codec_id              = 0;
+
+    avc_profile                 = 0;
+    avc_level                   = 0;
+    aac_profile                 = 0;
+    aac_sample_rate             = __SRS_AAC_SAMPLE_RATE_UNSET; // sample rate ignored
+    aac_channels                = 0;
+    avc_extra_size              = 0;
+    avc_extra_data              = NULL;
+    aac_extra_size              = 0;
+    aac_extra_data              = NULL;
+
+    sequenceParameterSetLength  = 0;
+    sequenceParameterSetNALUnit = NULL;
+    pictureParameterSetLength   = 0;
+    pictureParameterSetNALUnit  = NULL;
+
+    stream = new SrsStream();
+}
+
+SrsAvcAacCodec::~SrsAvcAacCodec()
+{
+    srs_freep(avc_extra_data);
+    srs_freep(aac_extra_data);
+
+    srs_freep(stream);
+    srs_freep(sequenceParameterSetNALUnit);
+    srs_freep(pictureParameterSetNALUnit);
+}
+
+int SrsAvcAacCodec::audio_aac_demux(char* data, int size, SrsCodecSample* sample)
+{
+    int ret = ERROR_SUCCESS;
+    
+    sample->is_video = false;
+    
+    if (!data || size <= 0) {
+        srs_trace("no audio present, ignore it.");
+        return ret;
+    }
+    
+    if ((ret = stream->initialize(data, size)) != ERROR_SUCCESS) {
+        return ret;
+    }
+
+    // audio decode
+    if (!stream->require(1)) {
+        ret = ERROR_HLS_DECODE_ERROR;
+        srs_error("audio codec decode sound_format failed. ret=%d", ret);
+        return ret;
+    }
+    
+    // @see: E.4.2 Audio Tags, video_file_format_spec_v10_1.pdf, page 76
+    int8_t sound_format = stream->read_1bytes();
+    
+    int8_t sound_type = sound_format & 0x01;
+    int8_t sound_size = (sound_format >> 1) & 0x01;
+    int8_t sound_rate = (sound_format >> 2) & 0x03;
+    sound_format = (sound_format >> 4) & 0x0f;
+    
+    audio_codec_id = sound_format;
+    sample->acodec = (SrsCodecAudio)audio_codec_id;
+
+    sample->sound_type = (SrsCodecAudioSoundType)sound_type;
+    sample->sound_rate = (SrsCodecAudioSampleRate)sound_rate;
+    sample->sound_size = (SrsCodecAudioSampleSize)sound_size;
+
+    // we support h.264+mp3 for hls.
+    if (audio_codec_id == SrsCodecAudioMP3) {
+        return ERROR_HLS_TRY_MP3;
+    }
+    
+    // only support aac
+    if (audio_codec_id != SrsCodecAudioAAC) {
+        ret = ERROR_HLS_DECODE_ERROR;
+        srs_error("audio codec only support mp3/aac codec. actual=%d, ret=%d", audio_codec_id, ret);
+        return ret;
+    }
+
+    if (!stream->require(1)) {
+        ret = ERROR_HLS_DECODE_ERROR;
+        srs_error("audio codec decode aac_packet_type failed. ret=%d", ret);
+        return ret;
+    }
+    
+    int8_t aac_packet_type = stream->read_1bytes();
+    sample->aac_packet_type = (SrsCodecAudioType)aac_packet_type;
+    
+    if (aac_packet_type == SrsCodecAudioTypeSequenceHeader) {
+        // AudioSpecificConfig
+        // 1.6.2.1 AudioSpecificConfig, in aac-mp4a-format-ISO_IEC_14496-3+2001.pdf, page 33.
+        aac_extra_size = stream->size() - stream->pos();
+        if (aac_extra_size > 0) {
+            srs_freep(aac_extra_data);
+            aac_extra_data = new char[aac_extra_size];
+            memcpy(aac_extra_data, stream->data() + stream->pos(), aac_extra_size);
+        }
+        
+        // only need to decode the first 2bytes:
+        // audioObjectType, aac_profile, 5bits.
+        // samplingFrequencyIndex, aac_sample_rate, 4bits.
+        // channelConfiguration, aac_channels, 4bits
+        if (!stream->require(2)) {
+            ret = ERROR_HLS_DECODE_ERROR;
+            srs_error("audio codec decode aac sequence header failed. ret=%d", ret);
+            return ret;
+        }
+        aac_profile = stream->read_1bytes();
+        aac_sample_rate = stream->read_1bytes();
+        
+        aac_channels = (aac_sample_rate >> 3) & 0x0f;
+        aac_sample_rate = ((aac_profile << 1) & 0x0e) | ((aac_sample_rate >> 7) & 0x01);
+        aac_profile = (aac_profile >> 3) & 0x1f;
+        
+        if (aac_profile == 0 || aac_profile == 0x1f) {
+            ret = ERROR_HLS_DECODE_ERROR;
+            srs_error("audio codec decode aac sequence header failed, "
+                "adts object=%d invalid. ret=%d", aac_profile, ret);
+            return ret;
+        }
+        
+        // TODO: FIXME: to support aac he/he-v2, see: ngx_rtmp_codec_parse_aac_header
+        // @see: https://github.com/winlinvip/nginx-rtmp-module/commit/3a5f9eea78fc8d11e8be922aea9ac349b9dcbfc2
+        // 
+        // donot force to LC, @see: https://github.com/winlinvip/simple-rtmp-server/issues/81
+        // the source will print the sequence header info.
+        //if (aac_profile > 3) {
+            // Mark all extended profiles as LC
+            // to make Android as happy as possible.
+            // @see: ngx_rtmp_hls_parse_aac_header
+            //aac_profile = 1;
+        //}
+    } else if (aac_packet_type == SrsCodecAudioTypeRawData) {
+        // ensure the sequence header demuxed
+        if (aac_extra_size <= 0 || !aac_extra_data) {
+            ret = ERROR_HLS_DECODE_ERROR;
+            srs_error("audio codec decode aac failed, sequence header not found. ret=%d", ret);
+            return ret;
+        }
+        
+        // Raw AAC frame data in UI8 []
+        // 6.3 Raw Data, aac-iso-13818-7.pdf, page 28
+        if ((ret = sample->add_sample_unit(stream->data() + stream->pos(), stream->size() - stream->pos())) != ERROR_SUCCESS) {
+            srs_error("audio codec add sample failed. ret=%d", ret);
+            return ret;
+        }
+    } else {
+        // ignored.
+    }
+    
+    // reset the sample rate by sequence header
+    if (aac_sample_rate != __SRS_AAC_SAMPLE_RATE_UNSET) {
+        static int aac_sample_rates[] = {
+            96000, 88200, 64000, 48000,
+            44100, 32000, 24000, 22050,
+            16000, 12000, 11025,  8000,
+            7350,     0,     0,    0
+        };
+        switch (aac_sample_rates[aac_sample_rate]) {
+            case 11025:
+                sample->sound_rate = SrsCodecAudioSampleRate11025;
+                break;
+            case 22050:
+                sample->sound_rate = SrsCodecAudioSampleRate22050;
+                break;
+            case 44100:
+                sample->sound_rate = SrsCodecAudioSampleRate44100;
+                break;
+            default:
+                break;
+        };
+    }
+    
+    srs_info("audio decoded, type=%d, codec=%d, asize=%d, rate=%d, format=%d, size=%d", 
+        sound_type, audio_codec_id, sound_size, sound_rate, sound_format, size);
+    
+    return ret;
+}
+
+int SrsAvcAacCodec::audio_mp3_demux(char* data, int size, SrsCodecSample* sample)
+{
+    int ret = ERROR_SUCCESS;
+
+    // we always decode aac then mp3.
+    srs_assert(sample->acodec == SrsCodecAudioMP3);
+    
+    // @see: E.4.2 Audio Tags, video_file_format_spec_v10_1.pdf, page 76
+    if (!data || size <= 1) {
+        srs_trace("no mp3 audio present, ignore it.");
+        return ret;
+    }
+
+    // mp3 payload.
+    if ((ret = sample->add_sample_unit(data + 1, size - 1)) != ERROR_SUCCESS) {
+        srs_error("audio codec add mp3 sample failed. ret=%d", ret);
+        return ret;
+    }
+    
+    srs_info("audio decoded, type=%d, codec=%d, asize=%d, rate=%d, format=%d, size=%d", 
+        sample->sound_type, audio_codec_id, sample->sound_size, sample->sound_rate, sample->acodec, size);
+    
+    return ret;
+}
+
+int SrsAvcAacCodec::video_avc_demux(char* data, int size, SrsCodecSample* sample)
+{
+    int ret = ERROR_SUCCESS;
+    
+    sample->is_video = true;
+    
+    if (!data || size <= 0) {
+        srs_trace("no video present, ignore it.");
+        return ret;
+    }
+    
+    if ((ret = stream->initialize(data, size)) != ERROR_SUCCESS) {
+        return ret;
+    }
+
+    // video decode
+    if (!stream->require(1)) {
+        ret = ERROR_HLS_DECODE_ERROR;
+        srs_error("video codec decode frame_type failed. ret=%d", ret);
+        return ret;
+    }
+    
+    // @see: E.4.3 Video Tags, video_file_format_spec_v10_1.pdf, page 78
+    int8_t frame_type = stream->read_1bytes();
+    int8_t codec_id = frame_type & 0x0f;
+    frame_type = (frame_type >> 4) & 0x0f;
+    
+    sample->frame_type = (SrsCodecVideoAVCFrame)frame_type;
+    
+    // ignore info frame without error,
+    // @see https://github.com/winlinvip/simple-rtmp-server/issues/288#issuecomment-69863909
+    if (sample->frame_type == SrsCodecVideoAVCFrameVideoInfoFrame) {
+        srs_warn("video codec igone the info frame, ret=%d", ret);
+        return ret;
+    }
+    
+    // only support h.264/avc
+    if (codec_id != SrsCodecVideoAVC) {
+        ret = ERROR_HLS_DECODE_ERROR;
+        srs_error("video codec only support video h.264/avc codec. actual=%d, ret=%d", codec_id, ret);
+        return ret;
+    }
+    video_codec_id = codec_id;
+    
+    if (!stream->require(4)) {
+        ret = ERROR_HLS_DECODE_ERROR;
+        srs_error("video codec decode avc_packet_type failed. ret=%d", ret);
+        return ret;
+    }
+    int8_t avc_packet_type = stream->read_1bytes();
+    int32_t composition_time = stream->read_3bytes();
+    
+    // pts = dts + cts.
+    sample->cts = composition_time;
+    sample->avc_packet_type = (SrsCodecVideoAVCType)avc_packet_type;
+    
+    if (avc_packet_type == SrsCodecVideoAVCTypeSequenceHeader) {
+        if ((ret = avc_demux_sps_pps(stream)) != ERROR_SUCCESS) {
+            return ret;
+        }
+    } else if (avc_packet_type == SrsCodecVideoAVCTypeNALU){
+        // ensure the sequence header demuxed
+        if (avc_extra_size <= 0 || !avc_extra_data) {
+            ret = ERROR_HLS_DECODE_ERROR;
+            srs_error("avc decode failed, sequence header not found. ret=%d", ret);
+            return ret;
+        }
+        
+        // One or more NALUs (Full frames are required)
+        // try  "AnnexB" from H.264-AVC-ISO_IEC_14496-10.pdf, page 211.
+        if ((ret = avc_demux_annexb_format(stream, sample)) != ERROR_SUCCESS) {
+            // stop try when system error.
+            if (ret != ERROR_HLS_AVC_TRY_OTHERS) {
+                srs_error("avc demux for annexb failed. ret=%d", ret);
+                return ret;
+            }
+            
+            // try "ISO Base Media File Format" from H.264-AVC-ISO_IEC_14496-15.pdf, page 20
+            if ((ret = avc_demux_ibmf_format(stream, sample)) != ERROR_SUCCESS) {
+                return ret;
+            }
+        }
+    } else {
+        // ignored.
+    }
+    
+    srs_info("video decoded, type=%d, codec=%d, avc=%d, time=%d, size=%d", 
+        frame_type, video_codec_id, avc_packet_type, composition_time, size);
+    
+    return ret;
+}
+
+int SrsAvcAacCodec::avc_demux_sps_pps(SrsStream* stream)
+{
+    int ret = ERROR_SUCCESS;
+    
+    // AVCDecoderConfigurationRecord
+    // 5.2.4.1.1 Syntax, H.264-AVC-ISO_IEC_14496-15.pdf, page 16
+    avc_extra_size = stream->size() - stream->pos();
+    if (avc_extra_size > 0) {
+        srs_freep(avc_extra_data);
+        avc_extra_data = new char[avc_extra_size];
+        memcpy(avc_extra_data, stream->data() + stream->pos(), avc_extra_size);
+    }
+    
+    if (!stream->require(6)) {
+        ret = ERROR_HLS_DECODE_ERROR;
+        srs_error("avc decode sequenc header failed. ret=%d", ret);
+        return ret;
+    }
+    //int8_t configurationVersion = stream->read_1bytes();
+    stream->read_1bytes();
+    //int8_t AVCProfileIndication = stream->read_1bytes();
+    avc_profile = stream->read_1bytes();
+    //int8_t profile_compatibility = stream->read_1bytes();
+    stream->read_1bytes();
+    //int8_t AVCLevelIndication = stream->read_1bytes();
+    avc_level = stream->read_1bytes();
+    
+    // parse the NALU size.
+    int8_t lengthSizeMinusOne = stream->read_1bytes();
+    lengthSizeMinusOne &= 0x03;
+    NAL_unit_length = lengthSizeMinusOne;
+    
+    // 5.3.4.2.1 Syntax, H.264-AVC-ISO_IEC_14496-15.pdf, page 16
+    // 5.2.4.1 AVC decoder configuration record
+    // 5.2.4.1.2 Semantics
+    // The value of this field shall be one of 0, 1, or 3 corresponding to a
+    // length encoded with 1, 2, or 4 bytes, respectively.
+    if (NAL_unit_length == 2) {
+        ret = ERROR_HLS_DECODE_ERROR;
+        srs_error("sps lengthSizeMinusOne should never be 2. ret=%d", ret);
+        return ret;
+    }
+    
+    // 1 sps
+    if (!stream->require(1)) {
+        ret = ERROR_HLS_DECODE_ERROR;
+        srs_error("avc decode sequenc header sps failed. ret=%d", ret);
+        return ret;
+    }
+    int8_t numOfSequenceParameterSets = stream->read_1bytes();
+    numOfSequenceParameterSets &= 0x1f;
+    if (numOfSequenceParameterSets != 1) {
+        ret = ERROR_HLS_DECODE_ERROR;
+        srs_error("avc decode sequenc header sps failed. ret=%d", ret);
+        return ret;
+    }
+    if (!stream->require(2)) {
+        ret = ERROR_HLS_DECODE_ERROR;
+        srs_error("avc decode sequenc header sps size failed. ret=%d", ret);
+        return ret;
+    }
+    sequenceParameterSetLength = stream->read_2bytes();
+    if (!stream->require(sequenceParameterSetLength)) {
+        ret = ERROR_HLS_DECODE_ERROR;
+        srs_error("avc decode sequenc header sps data failed. ret=%d", ret);
+        return ret;
+    }
+    if (sequenceParameterSetLength > 0) {
+        srs_freep(sequenceParameterSetNALUnit);
+        sequenceParameterSetNALUnit = new char[sequenceParameterSetLength];
+        memcpy(sequenceParameterSetNALUnit, stream->data() + stream->pos(), sequenceParameterSetLength);
+        stream->skip(sequenceParameterSetLength);
+    }
+    // 1 pps
+    if (!stream->require(1)) {
+        ret = ERROR_HLS_DECODE_ERROR;
+        srs_error("avc decode sequenc header pps failed. ret=%d", ret);
+        return ret;
+    }
+    int8_t numOfPictureParameterSets = stream->read_1bytes();
+    numOfPictureParameterSets &= 0x1f;
+    if (numOfPictureParameterSets != 1) {
+        ret = ERROR_HLS_DECODE_ERROR;
+        srs_error("avc decode sequenc header pps failed. ret=%d", ret);
+        return ret;
+    }
+    if (!stream->require(2)) {
+        ret = ERROR_HLS_DECODE_ERROR;
+        srs_error("avc decode sequenc header pps size failed. ret=%d", ret);
+        return ret;
+    }
+    pictureParameterSetLength = stream->read_2bytes();
+    if (!stream->require(pictureParameterSetLength)) {
+        ret = ERROR_HLS_DECODE_ERROR;
+        srs_error("avc decode sequenc header pps data failed. ret=%d", ret);
+        return ret;
+    }
+    if (pictureParameterSetLength > 0) {
+        srs_freep(pictureParameterSetNALUnit);
+        pictureParameterSetNALUnit = new char[pictureParameterSetLength];
+        memcpy(pictureParameterSetNALUnit, stream->data() + stream->pos(), pictureParameterSetLength);
+        stream->skip(pictureParameterSetLength);
+    }
+    
+    return ret;
+}
+
+int SrsAvcAacCodec::avc_demux_annexb_format(SrsStream* stream, SrsCodecSample* sample)
+{
+    int ret = ERROR_SUCCESS;
+    
+    // not annexb, try others
+    if (!srs_avc_startswith_annexb(stream, NULL)) {
+        return ERROR_HLS_AVC_TRY_OTHERS;
+    }
+    
+    // AnnexB
+    // B.1.1 Byte stream NAL unit syntax,
+    // H.264-AVC-ISO_IEC_14496-10.pdf, page 211.
+    while (!stream->empty()) {
+        // find start code
+        int nb_start_code = 0;
+        if (!srs_avc_startswith_annexb(stream, &nb_start_code)) {
+            return ret;
+        }
+
+        // skip the start code.
+        if (nb_start_code > 0) {
+            stream->skip(nb_start_code);
+        }
+        
+        // the NALU start bytes.
+        char* p = stream->data() + stream->pos();
+        
+        // get the last matched NALU
+        while (!stream->empty()) {
+            if (srs_avc_startswith_annexb(stream, NULL)) {
+                break;
+            }
+            
+            stream->skip(1);
+        }
+        
+        char* pp = stream->data() + stream->pos();
+        
+        // skip the empty.
+        if (pp - p <= 0) {
+            continue;
+        }
+        
+        // got the NALU.
+        if ((ret = sample->add_sample_unit(p, pp - p)) != ERROR_SUCCESS) {
+            srs_error("annexb add video sample failed. ret=%d", ret);
+            return ret;
+        }
+    }
+    
+    return ret;
+}
+
+int SrsAvcAacCodec::avc_demux_ibmf_format(SrsStream* stream, SrsCodecSample* sample)
+{
+    int ret = ERROR_SUCCESS;
+    
+    int PictureLength = stream->size() - stream->pos();
+    
+    // 5.3.4.2.1 Syntax, H.264-AVC-ISO_IEC_14496-15.pdf, page 16
+    // 5.2.4.1 AVC decoder configuration record
+    // 5.2.4.1.2 Semantics
+    // The value of this field shall be one of 0, 1, or 3 corresponding to a
+    // length encoded with 1, 2, or 4 bytes, respectively.
+    srs_assert(NAL_unit_length != 2);
+    
+    // 5.3.4.2.1 Syntax, H.264-AVC-ISO_IEC_14496-15.pdf, page 20
+    for (int i = 0; i < PictureLength;) {
+        // unsigned int((NAL_unit_length+1)*8) NALUnitLength;
+        if (!stream->require(NAL_unit_length + 1)) {
+            ret = ERROR_HLS_DECODE_ERROR;
+            srs_error("avc decode NALU size failed. ret=%d", ret);
+            return ret;
+        }
+        int32_t NALUnitLength = 0;
+        if (NAL_unit_length == 3) {
+            NALUnitLength = stream->read_4bytes();
+        } else if (NAL_unit_length == 1) {
+            NALUnitLength = stream->read_2bytes();
+        } else {
+            NALUnitLength = stream->read_1bytes();
+        }
+        
+        // maybe stream is invalid format.
+        // see: https://github.com/winlinvip/simple-rtmp-server/issues/183
+        if (NALUnitLength < 0) {
+            ret = ERROR_HLS_DECODE_ERROR;
+            srs_error("maybe stream is AnnexB format. ret=%d", ret);
+            return ret;
+        }
+        
+        // NALUnit
+        if (!stream->require(NALUnitLength)) {
+            ret = ERROR_HLS_DECODE_ERROR;
+            srs_error("avc decode NALU data failed. ret=%d", ret);
+            return ret;
+        }
+        // 7.3.1 NAL unit syntax, H.264-AVC-ISO_IEC_14496-10.pdf, page 44.
+        if ((ret = sample->add_sample_unit(stream->data() + stream->pos(), NALUnitLength)) != ERROR_SUCCESS) {
+            srs_error("avc add video sample failed. ret=%d", ret);
+            return ret;
+        }
+        stream->skip(NALUnitLength);
+        
+        i += NAL_unit_length + 1 + NALUnitLength;
+    }
+    
+    return ret;
+}
+

@@ -30,6 +30,10 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 
 #include <srs_core.hpp>
 #include <srs_core.hpp>
 
 
+#include <string>
+
+class SrsStream;
+
 // AACPacketType IF SoundFormat == 10 UI8
 // AACPacketType IF SoundFormat == 10 UI8
 // The following values are defined:
 // The following values are defined:
 //     0 = AAC sequence header
 //     0 = AAC sequence header
@@ -212,4 +216,263 @@ public:
     static bool audio_is_aac(char* data, int size);
     static bool audio_is_aac(char* data, int size);
 };
 };
 
 
+/**
+* the public data, event HLS disable, others can use it.
+*/
+/**
+* the flv sample rate map
+*/
+extern int flv_sample_rates[];
+
+/**
+* the aac sample rate map
+*/
+extern int aac_sample_rates[];
+
+#define __SRS_SRS_MAX_CODEC_SAMPLE 128
+#define __SRS_AAC_SAMPLE_RATE_UNSET 15
+
+// in ms, for HLS aac flush the audio
+#define SRS_CONF_DEFAULT_AAC_DELAY 100
+
+// max PES packets size to flush the video.
+#define SRS_AUTO_HLS_AUDIO_CACHE_SIZE 1024 * 1024
+
+/**
+* the FLV/RTMP supported audio sample size.
+* Size of each audio sample. This parameter only pertains to
+* uncompressed formats. Compressed formats always decode
+* to 16 bits internally.
+* 0 = 8-bit samples
+* 1 = 16-bit samples
+*/
+enum SrsCodecAudioSampleSize
+{
+    // set to the max value to reserved, for array map.
+    SrsCodecAudioSampleSizeReserved                 = 2,
+    
+    SrsCodecAudioSampleSize8bit                     = 0,
+    SrsCodecAudioSampleSize16bit                    = 1,
+};
+
+/**
+* the FLV/RTMP supported audio sound type/channel.
+* Mono or stereo sound
+* 0 = Mono sound
+* 1 = Stereo sound
+*/
+enum SrsCodecAudioSoundType
+{
+    // set to the max value to reserved, for array map.
+    SrsCodecAudioSoundTypeReserved                  = 2, 
+    
+    SrsCodecAudioSoundTypeMono                      = 0,
+    SrsCodecAudioSoundTypeStereo                    = 1,
+};
+
+/**
+* the codec sample unit.
+* for h.264 video packet, a NALU is a sample unit.
+* for aac raw audio packet, a NALU is the entire aac raw data.
+* for sequence header, it's not a sample unit.
+*/
+class SrsCodecSampleUnit
+{
+public:
+    /**
+    * the sample bytes is directly ptr to packet bytes,
+    * user should never use it when packet destroyed.
+    */
+    int size;
+    char* bytes;
+public:
+    SrsCodecSampleUnit();
+    virtual ~SrsCodecSampleUnit();
+};
+
+/**
+* the samples in the flv audio/video packet.
+* the sample used to analysis a video/audio packet,
+* split the h.264 NALUs to buffers, or aac raw data to a buffer,
+* and decode the video/audio specified infos.
+* 
+* the sample unit:
+*       a video packet codec in h.264 contains many NALUs, each is a sample unit.
+*       a audio packet codec in aac is a sample unit.
+* @remark, the video/audio sequence header is not sample unit,
+*       all sequence header stores as extra data, 
+*       @see SrsAvcAacCodec.avc_extra_data and SrsAvcAacCodec.aac_extra_data
+* @remark, user must clear all samples before decode a new video/audio packet.
+*/
+class SrsCodecSample
+{
+public:
+    /**
+    * each audio/video raw data packet will dumps to one or multiple buffers,
+    * the buffers will write to hls and clear to reset.
+    * generally, aac audio packet corresponding to one buffer,
+    * where avc/h264 video packet may contains multiple buffer.
+    */
+    int nb_sample_units;
+    SrsCodecSampleUnit sample_units[__SRS_SRS_MAX_CODEC_SAMPLE];
+public:
+    /**
+    * whether the sample is video sample which demux from video packet.
+    */
+    bool is_video;
+    /**
+    * CompositionTime, video_file_format_spec_v10_1.pdf, page 78.
+    * cts = pts - dts, where dts = flvheader->timestamp.
+    */
+    int32_t cts;
+public:
+    // video specified
+    SrsCodecVideoAVCFrame frame_type;
+    SrsCodecVideoAVCType avc_packet_type;
+public:
+    // audio specified
+    SrsCodecAudio acodec;
+    // audio aac specified.
+    SrsCodecAudioSampleRate sound_rate;
+    SrsCodecAudioSampleSize sound_size;
+    SrsCodecAudioSoundType sound_type;
+    SrsCodecAudioType aac_packet_type;
+public:
+    SrsCodecSample();
+    virtual ~SrsCodecSample();
+public:
+    /**
+    * clear all samples.
+    * the sample units never copy the bytes, it directly use the ptr,
+    * so when video/audio packet is destroyed, the sample must be clear.
+    * in a word, user must clear sample before demux it.
+    * @remark demux sample use SrsAvcAacCodec.audio_aac_demux or video_avc_demux.
+    */
+    void clear();
+    /**
+    * add the a sample unit, it's a h.264 NALU or aac raw data.
+    * the sample unit directly use the ptr of packet bytes,
+    * so user must never use sample unit when packet is destroyed.
+    * in a word, user must clear sample before demux it.
+    */
+    int add_sample_unit(char* bytes, int size);
+};
+
+/**
+* the h264/avc and aac codec, for media stream.
+*
+* to demux the FLV/RTMP video/audio packet to sample,
+* add each NALUs of h.264 as a sample unit to sample,
+* while the entire aac raw data as a sample unit.
+*
+* for sequence header,
+* demux it and save it in the avc_extra_data and aac_extra_data,
+* 
+* for the codec info, such as audio sample rate,
+* decode from FLV/RTMP header, then use codec info in sequence 
+* header to override it.
+*/
+class SrsAvcAacCodec
+{
+private:
+    SrsStream* stream;
+public:
+    /**
+    * metadata specified
+    */
+    int             duration;
+    int             width;
+    int             height;
+    int             frame_rate;
+    // @see: SrsCodecVideo
+    int             video_codec_id;
+    int             video_data_rate; // in bps
+    // @see: SrsCod ecAudioType
+    int             audio_codec_id;
+    int             audio_data_rate; // in bps
+public:
+    /**
+    * video specified
+    */
+    // profile_idc, H.264-AVC-ISO_IEC_14496-10.pdf, page 45.
+    u_int8_t        avc_profile; 
+    // level_idc, H.264-AVC-ISO_IEC_14496-10.pdf, page 45.
+    u_int8_t        avc_level; 
+    // lengthSizeMinusOne, H.264-AVC-ISO_IEC_14496-15.pdf, page 16
+    int8_t          NAL_unit_length;
+    u_int16_t       sequenceParameterSetLength;
+    char*           sequenceParameterSetNALUnit;
+    u_int16_t       pictureParameterSetLength;
+    char*           pictureParameterSetNALUnit;
+public:
+    /**
+    * audio specified
+    * audioObjectType, in 1.6.2.1 AudioSpecificConfig, page 33,
+    * 1.5.1.1 Audio object type definition, page 23,
+    *           in aac-mp4a-format-ISO_IEC_14496-3+2001.pdf.
+    */
+    u_int8_t        aac_profile; 
+    /**
+    * samplingFrequencyIndex
+    */
+    u_int8_t        aac_sample_rate;
+    /**
+    * channelConfiguration
+    */
+    u_int8_t        aac_channels;
+public:
+    /**
+    * the avc extra data, the AVC sequence header,
+    * without the flv codec header,
+    * @see: ffmpeg, AVCodecContext::extradata
+    */
+    int             avc_extra_size;
+    char*           avc_extra_data;
+    /**
+    * the aac extra data, the AAC sequence header,
+    * without the flv codec header,
+    * @see: ffmpeg, AVCodecContext::extradata
+    */
+    int             aac_extra_size;
+    char*           aac_extra_data;
+public:
+    SrsAvcAacCodec();
+    virtual ~SrsAvcAacCodec();
+// the following function used for hls to build the sample and codec.
+public:
+    /**
+    * demux the audio packet in aac codec.
+    * the packet mux in FLV/RTMP format defined in flv specification.
+    * demux the audio speicified data(sound_format, sound_size, ...) to sample.
+    * demux the aac specified data(aac_profile, ...) to codec from sequence header.
+    * demux the aac raw to sample units.
+    */
+    virtual int audio_aac_demux(char* data, int size, SrsCodecSample* sample);
+    virtual int audio_mp3_demux(char* data, int size, SrsCodecSample* sample);
+    /**
+    * demux the video packet in h.264 codec.
+    * the packet mux in FLV/RTMP format defined in flv specification.
+    * demux the video specified data(frame_type, codec_id, ...) to sample.
+    * demux the h.264 sepcified data(avc_profile, ...) to codec from sequence header.
+    * demux the h.264 NALUs to sampe units.
+    */
+    virtual int video_avc_demux(char* data, int size, SrsCodecSample* sample);
+private:
+    /**
+    * when avc packet type is SrsCodecVideoAVCTypeSequenceHeader,
+    * decode the sps and pps.
+    */
+    virtual int avc_demux_sps_pps(SrsStream* stream);
+    /**
+    * demux the avc NALU in "AnnexB" 
+    * from H.264-AVC-ISO_IEC_14496-10.pdf, page 211.
+    */
+    virtual int avc_demux_annexb_format(SrsStream* stream, SrsCodecSample* sample);
+    /**
+    * demux the avc NALU in "ISO Base Media File Format" 
+    * from H.264-AVC-ISO_IEC_14496-15.pdf, page 20
+    */
+    virtual int avc_demux_ibmf_format(SrsStream* stream, SrsCodecSample* sample);
+};
+
 #endif
 #endif

@@ -35,7 +35,7 @@ using namespace std;
 #include <srs_kernel_log.hpp>
 #include <srs_kernel_log.hpp>
 #include <srs_kernel_error.hpp>
 #include <srs_kernel_error.hpp>
 #include <srs_kernel_file.hpp>
 #include <srs_kernel_file.hpp>
-#include <srs_kernel_avc.hpp>
+#include <srs_kernel_codec.hpp>
 #include <srs_kernel_buffer.hpp>
 #include <srs_kernel_buffer.hpp>
 #include <srs_kernel_utility.hpp>
 #include <srs_kernel_utility.hpp>
 #include <srs_kernel_stream.hpp>
 #include <srs_kernel_stream.hpp>

@@ -1520,7 +1520,6 @@ int srs_h264_write_raw_frames(srs_rtmp_t rtmp,
     while (!context->h264_raw_stream.empty()) {
     while (!context->h264_raw_stream.empty()) {
         char* frame = NULL;
         char* frame = NULL;
         int frame_size = 0;
         int frame_size = 0;
-        bool got_sps_pps = false;
         if ((ret = context->avc_raw.annexb_demux(&context->h264_raw_stream, &frame, &frame_size)) != ERROR_SUCCESS) {
         if ((ret = context->avc_raw.annexb_demux(&context->h264_raw_stream, &frame, &frame_size)) != ERROR_SUCCESS) {
             return ret;
             return ret;
         }
         }