fix #467, support write log to kafka. 3.0.6

winlin
Commit 9a473902530e35b41f6e0acfc4fe364d4d74d2d3 9a473902 1 parent 76cd3f87
README.md
trunk/src/core/srs_core.hpp
trunk/src/kernel/srs_kernel_buffer.cpp
trunk/src/kernel/srs_kernel_buffer.hpp
trunk/src/kernel/srs_kernel_ts.cpp
trunk/src/kernel/srs_kernel_utility.cpp
trunk/src/kernel/srs_kernel_utility.hpp
trunk/src/protocol/srs_kafka_stack.cpp
trunk/src/protocol/srs_kafka_stack.hpp
--- a/README.md
查看文件 @9a47390
+++ b/README.md
查看文件 @9a47390
@@ -259,7 +259,7 @@ The `features`, `compare`, `release` and `performance` of SRS.
 1. Support NGINX-RTMP style EXEC, read [#367][bug #367].
 1. Support NGINX-RTMP style dvr control module, read [#459][bug #459].
 1. Support HTTP Security Raw Api, read [#459][bug #459], [#470][bug #470], [#319][bug #319].
-1. [dev]Support Integration with Kafka/Spark Big-Data system, read [#467][bug #467].
+1. Support Integration with Kafka/Spark Big-Data system, read [#467][bug #467].
 1. [plan]Support Origin Cluster for Load Balance and Fault Tolarence, read [#464][bug #464], [RTMP 302][bug #92].
 1. [plan]Support H.265, push RTMP with H.265, delivery in HLS, read [#465][bug #465].
 1. [plan]Support MPEG-DASH, the future streaming protocol, read [#299][bug #299].
@@ -386,6 +386,7 @@ Remark:
 ### History
+* v3.0, 2015-10-23, fix [#467][bug #467], support write log to kafka. 3.0.6
 * v3.0, 2015-10-20, fix [#502][bug #502], support snapshot with http-callback or transcoder. 3.0.5
 * v3.0, 2015-09-19, support amf0 and json to convert with each other.
 * v3.0, 2015-09-19, json objects support dumps to string.
@@ -1279,6 +1280,7 @@ Winlin
 [bug #466]: https://github.com/simple-rtmp-server/srs/issues/466
 [bug #468]: https://github.com/simple-rtmp-server/srs/issues/468
 [bug #502]: https://github.com/simple-rtmp-server/srs/issues/502
+[bug #467]: https://github.com/simple-rtmp-server/srs/issues/467
 [bug #xxxxxxx]: https://github.com/simple-rtmp-server/srs/issues/xxxxxxx
 [r2.0a2]: https://github.com/simple-rtmp-server/srs/releases/tag/v2.0-a2
--- a/trunk/src/core/srs_core.hpp
查看文件 @9a47390
+++ b/trunk/src/core/srs_core.hpp
查看文件 @9a47390
@@ -31,7 +31,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 // current release version
 #define VERSION_MAJOR       3
 #define VERSION_MINOR       0
-#define VERSION_REVISION    5
+#define VERSION_REVISION    6
 // server info.
 #define RTMP_SIG_SRS_KEY "SRS"
--- a/trunk/src/kernel/srs_kernel_buffer.cpp
查看文件 @9a47390
+++ b/trunk/src/kernel/srs_kernel_buffer.cpp
查看文件 @9a47390
@@ -39,17 +39,27 @@ ISrsCodec::~ISrsCodec()
 SrsBuffer::SrsBuffer()
 {
-    p = bytes = NULL;
-    nb_bytes = 0;
+    set_value(NULL, 0);
+}
-    // TODO: support both little and big endian.
-    srs_assert(srs_is_little_endian());
+SrsBuffer::SrsBuffer(char* b, int nb_b)
+{
+    set_value(b, nb_b);
 }
 SrsBuffer::~SrsBuffer()
 {
 }
+void SrsBuffer::set_value(char* b, int nb_b)
+{
+    p = bytes = b;
+    nb_bytes = nb_b;
+    
+    // TODO: support both little and big endian.
+    srs_assert(srs_is_little_endian());
+}
+
 int SrsBuffer::initialize(char* b, int nb)
 {
     int ret = ERROR_SUCCESS;
--- a/trunk/src/kernel/srs_kernel_buffer.hpp
查看文件 @9a47390
+++ b/trunk/src/kernel/srs_kernel_buffer.hpp
查看文件 @9a47390
@@ -98,7 +98,10 @@ private:
     int nb_bytes;
 public:
     SrsBuffer();
+    SrsBuffer(char* b, int nb_b);
     virtual ~SrsBuffer();
+private:
+    virtual void set_value(char* b, int nb_b);
 public:
     /**
     * initialize the stream from bytes.
--- a/trunk/src/kernel/srs_kernel_ts.cpp
查看文件 @9a47390
+++ b/trunk/src/kernel/srs_kernel_ts.cpp
查看文件 @9a47390
@@ -2141,7 +2141,7 @@ int SrsTsPayloadPSI::decode(SrsBuffer* stream, SrsTsMessage** /*ppmsg*/)
     CRC_32 = stream->read_4bytes();
     // verify crc32.
-    int32_t crc32 = srs_crc32(ppat, stream->pos() - pat_pos - 4);
+    int32_t crc32 = srs_crc32_mpegts(ppat, stream->pos() - pat_pos - 4);
     if (crc32 != CRC_32) {
         ret = ERROR_STREAM_CASTER_TS_CRC32;
         srs_error("ts: verify PSI crc32 failed. ret=%d", ret);
@@ -2238,7 +2238,7 @@ int SrsTsPayloadPSI::encode(SrsBuffer* stream)
         srs_error("ts: mux PSI crc32 failed. ret=%d", ret);
         return ret;
     }
-    CRC_32 = srs_crc32(ppat, stream->pos() - pat_pos);
+    CRC_32 = srs_crc32_mpegts(ppat, stream->pos() - pat_pos);
     stream->write_4bytes(CRC_32);
     return ret;
--- a/trunk/src/kernel/srs_kernel_utility.cpp
查看文件 @9a47390
+++ b/trunk/src/kernel/srs_kernel_utility.cpp
查看文件 @9a47390
@@ -535,7 +535,10 @@ bool srs_aac_startswith_adts(SrsBuffer* stream)
     return true;
 }
-/*
+// @see http://www.stmc.edu.hk/~vincent/ffmpeg_0.4.9-pre1/libavformat/mpegtsenc.c
+unsigned int __mpegts_crc32(const u_int8_t *data, int len)
+{
+    /*
      * MPEG2 transport stream (aka DVB) mux
      * Copyright (c) 2003 Fabrice Bellard.
      *
@@ -553,7 +556,7 @@ bool srs_aac_startswith_adts(SrsBuffer* stream)
      * License along with this library; if not, write to the Free Software
      * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
      */
-static const u_int32_t crc_table[256] = {
+    static const u_int32_t table[256] = {
         0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, 0x130476dc, 0x17c56b6b,
         0x1a864db2, 0x1e475005, 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61,
         0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd, 0x4c11db70, 0x48d0c6c7,
@@ -597,23 +600,98 @@ static const u_int32_t crc_table[256] = {
         0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, 0x9abc8bd5, 0x9e7d9662,
         0x933eb0bb, 0x97ffad0c, 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668,
         0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4
-};
+    };
-// @see http://www.stmc.edu.hk/~vincent/ffmpeg_0.4.9-pre1/libavformat/mpegtsenc.c
-unsigned int mpegts_crc32(const u_int8_t *data, int len)
-{
-    register int i;
-    unsigned int crc = 0xffffffff;
+    u_int32_t crc = 0xffffffff;
-    for (i=0; i<len; i++)
-        crc = (crc << 8) ^ crc_table[((crc >> 24) ^ *data++) & 0xff];
+    for (int i=0; i<len; i++) {
+        crc = (crc << 8) ^ table[((crc >> 24) ^ *data++) & 0xff];
+    }
     return crc;
 }
-u_int32_t srs_crc32(const void* buf, int size)
+// @see https://github.com/ETrun/crc32/blob/master/crc32.c
+u_int32_t __crc32_ieee(u_int32_t init, const u_int8_t* buf, size_t nb_buf)
+{
+    /*----------------------------------------------------------------------------*\
+     *  CRC-32 version 2.0.0 by Craig Bruce, 2006-04-29.
+     *
+     *  This program generates the CRC-32 values for the files named in the
+     *  command-line arguments.  These are the same CRC-32 values used by GZIP,
+     *  PKZIP, and ZMODEM.  The Crc32_ComputeBuf() can also be detached and
+     *  used independently.
+     *
+     *  THIS PROGRAM IS PUBLIC-DOMAIN SOFTWARE.
+     *
+     *  Based on the byte-oriented implementation "File Verification Using CRC"
+     *  by Mark R. Nelson in Dr. Dobb's Journal, May 1992, pp. 64-67.
+     *
+     *  v1.0.0: original release.
+     *  v1.0.1: fixed printf formats.
+     *  v1.0.2: fixed something else.
+     *  v1.0.3: replaced CRC constant table by generator function.
+     *  v1.0.4: reformatted code, made ANSI C.  1994-12-05.
+     *  v2.0.0: rewrote to use memory buffer & static table, 2006-04-29.
+     *  v2.1.0: modified by Nico, 2013-04-20
+     \*----------------------------------------------------------------------------*/
+    static const u_int32_t table[256] = {
+        0x00000000,0x77073096,0xEE0E612C,0x990951BA,0x076DC419,0x706AF48F,0xE963A535,
+        0x9E6495A3,0x0EDB8832,0x79DCB8A4,0xE0D5E91E,0x97D2D988,0x09B64C2B,0x7EB17CBD,
+        0xE7B82D07,0x90BF1D91,0x1DB71064,0x6AB020F2,0xF3B97148,0x84BE41DE,0x1ADAD47D,
+        0x6DDDE4EB,0xF4D4B551,0x83D385C7,0x136C9856,0x646BA8C0,0xFD62F97A,0x8A65C9EC,
+        0x14015C4F,0x63066CD9,0xFA0F3D63,0x8D080DF5,0x3B6E20C8,0x4C69105E,0xD56041E4,
+        0xA2677172,0x3C03E4D1,0x4B04D447,0xD20D85FD,0xA50AB56B,0x35B5A8FA,0x42B2986C,
+        0xDBBBC9D6,0xACBCF940,0x32D86CE3,0x45DF5C75,0xDCD60DCF,0xABD13D59,0x26D930AC,
+        0x51DE003A,0xC8D75180,0xBFD06116,0x21B4F4B5,0x56B3C423,0xCFBA9599,0xB8BDA50F,
+        0x2802B89E,0x5F058808,0xC60CD9B2,0xB10BE924,0x2F6F7C87,0x58684C11,0xC1611DAB,
+        0xB6662D3D,0x76DC4190,0x01DB7106,0x98D220BC,0xEFD5102A,0x71B18589,0x06B6B51F,
+        0x9FBFE4A5,0xE8B8D433,0x7807C9A2,0x0F00F934,0x9609A88E,0xE10E9818,0x7F6A0DBB,
+        0x086D3D2D,0x91646C97,0xE6635C01,0x6B6B51F4,0x1C6C6162,0x856530D8,0xF262004E,
+        0x6C0695ED,0x1B01A57B,0x8208F4C1,0xF50FC457,0x65B0D9C6,0x12B7E950,0x8BBEB8EA,
+        0xFCB9887C,0x62DD1DDF,0x15DA2D49,0x8CD37CF3,0xFBD44C65,0x4DB26158,0x3AB551CE,
+        0xA3BC0074,0xD4BB30E2,0x4ADFA541,0x3DD895D7,0xA4D1C46D,0xD3D6F4FB,0x4369E96A,
+        0x346ED9FC,0xAD678846,0xDA60B8D0,0x44042D73,0x33031DE5,0xAA0A4C5F,0xDD0D7CC9,
+        0x5005713C,0x270241AA,0xBE0B1010,0xC90C2086,0x5768B525,0x206F85B3,0xB966D409,
+        0xCE61E49F,0x5EDEF90E,0x29D9C998,0xB0D09822,0xC7D7A8B4,0x59B33D17,0x2EB40D81,
+        0xB7BD5C3B,0xC0BA6CAD,0xEDB88320,0x9ABFB3B6,0x03B6E20C,0x74B1D29A,0xEAD54739,
+        0x9DD277AF,0x04DB2615,0x73DC1683,0xE3630B12,0x94643B84,0x0D6D6A3E,0x7A6A5AA8,
+        0xE40ECF0B,0x9309FF9D,0x0A00AE27,0x7D079EB1,0xF00F9344,0x8708A3D2,0x1E01F268,
+        0x6906C2FE,0xF762575D,0x806567CB,0x196C3671,0x6E6B06E7,0xFED41B76,0x89D32BE0,
+        0x10DA7A5A,0x67DD4ACC,0xF9B9DF6F,0x8EBEEFF9,0x17B7BE43,0x60B08ED5,0xD6D6A3E8,
+        0xA1D1937E,0x38D8C2C4,0x4FDFF252,0xD1BB67F1,0xA6BC5767,0x3FB506DD,0x48B2364B,
+        0xD80D2BDA,0xAF0A1B4C,0x36034AF6,0x41047A60,0xDF60EFC3,0xA867DF55,0x316E8EEF,
+        0x4669BE79,0xCB61B38C,0xBC66831A,0x256FD2A0,0x5268E236,0xCC0C7795,0xBB0B4703,
+        0x220216B9,0x5505262F,0xC5BA3BBE,0xB2BD0B28,0x2BB45A92,0x5CB36A04,0xC2D7FFA7,
+        0xB5D0CF31,0x2CD99E8B,0x5BDEAE1D,0x9B64C2B0,0xEC63F226,0x756AA39C,0x026D930A,
+        0x9C0906A9,0xEB0E363F,0x72076785,0x05005713,0x95BF4A82,0xE2B87A14,0x7BB12BAE,
+        0x0CB61B38,0x92D28E9B,0xE5D5BE0D,0x7CDCEFB7,0x0BDBDF21,0x86D3D2D4,0xF1D4E242,
+        0x68DDB3F8,0x1FDA836E,0x81BE16CD,0xF6B9265B,0x6FB077E1,0x18B74777,0x88085AE6,
+        0xFF0F6A70,0x66063BCA,0x11010B5C,0x8F659EFF,0xF862AE69,0x616BFFD3,0x166CCF45,
+        0xA00AE278,0xD70DD2EE,0x4E048354,0x3903B3C2,0xA7672661,0xD06016F7,0x4969474D,
+        0x3E6E77DB,0xAED16A4A,0xD9D65ADC,0x40DF0B66,0x37D83BF0,0xA9BCAE53,0xDEBB9EC5,
+        0x47B2CF7F,0x30B5FFE9,0xBDBDF21C,0xCABAC28A,0x53B39330,0x24B4A3A6,0xBAD03605,
+        0xCDD70693,0x54DE5729,0x23D967BF,0xB3667A2E,0xC4614AB8,0x5D681B02,0x2A6F2B94,
+        0xB40BBE37,0xC30C8EA1,0x5A05DF1B,0x2D02EF8D
+    };
+    
+    u_int32_t crc = init ^ 0xFFFFFFFF;
+    
+    for (int i = 0; i < nb_buf; i++) {
+        crc = table[(crc ^ buf[i]) & 0xff] ^ (crc >> 8);
+    }
+    
+    return crc^0xFFFFFFFF;
+}
+
+u_int32_t srs_crc32_mpegts(const void* buf, int size)
+{
+    return __mpegts_crc32((const u_int8_t*)buf, size);
+}
+    
+u_int32_t srs_crc32_ieee(const void* buf, int size, u_int32_t previous)
 {
-    return mpegts_crc32((const u_int8_t*)buf, size);
+    return __crc32_ieee(previous, (const u_int8_t*)buf, size);
 }
 /*
--- a/trunk/src/kernel/srs_kernel_utility.hpp
查看文件 @9a47390
+++ b/trunk/src/kernel/srs_kernel_utility.hpp
查看文件 @9a47390
@@ -122,9 +122,14 @@ extern bool srs_avc_startswith_annexb(SrsBuffer* stream, int* pnb_start_code = N
 extern bool srs_aac_startswith_adts(SrsBuffer* stream);
 /**
-* cacl the crc32 of bytes in buf.
-*/
-extern u_int32_t srs_crc32(const void* buf, int size);
+* cacl the crc32 of bytes in buf, for ffmpeg.
+ */
+extern u_int32_t srs_crc32_mpegts(const void* buf, int size);
+
+/**
+ * calc the crc32 of bytes in buf by IEEE, for zip.
+ */
+extern u_int32_t srs_crc32_ieee(const void* buf, int size, u_int32_t previous = 0);
 /**
 * Decode a base64-encoded string.
--- a/trunk/src/protocol/srs_kafka_stack.cpp
查看文件 @9a47390
+++ b/trunk/src/protocol/srs_kafka_stack.cpp
查看文件 @9a47390
@@ -158,20 +158,30 @@ int SrsKafkaString::decode(SrsBuffer* buf)
 SrsKafkaBytes::SrsKafkaBytes()
 {
     _size = -1;
-    data = NULL;
+    _data = NULL;
 }
 SrsKafkaBytes::SrsKafkaBytes(const char* v, int nb_v)
 {
     _size = -1;
-    data = NULL;
+    _data = NULL;
     set_value(v, nb_v);
 }
 SrsKafkaBytes::~SrsKafkaBytes()
 {
-    srs_freep(data);
+    srs_freep(_data);
+}
+
+char* SrsKafkaBytes::data()
+{
+    return _data;
+}
+
+int SrsKafkaBytes::size()
+{
+    return _size;
 }
 bool SrsKafkaBytes::null()
@@ -192,14 +202,29 @@ void SrsKafkaBytes::set_value(string v)
 void SrsKafkaBytes::set_value(const char* v, int nb_v)
 {
     // free previous data.
-    srs_freep(data);
+    srs_freep(_data);
     // copy new value to data.
     _size = (int16_t)nb_v;
     srs_assert(_size > 0);
-    data = new char[_size];
-    memcpy(data, v, _size);
+    _data = new char[_size];
+    memcpy(_data, v, _size);
+}
+
+u_int32_t SrsKafkaBytes::crc32(u_int32_t previous)
+{
+    char bsize[4];
+    SrsBuffer(bsize, 4).write_4bytes(_size);
+    
+    if (_size <= 0) {
+        return srs_crc32_ieee(bsize, 4, previous);
+    }
+    
+    u_int32_t crc = srs_crc32_ieee(bsize, 4, previous);
+    crc = srs_crc32_ieee(_data, _size, crc);
+    
+    return crc;
 }
 int SrsKafkaBytes::nb_bytes()
@@ -227,7 +252,7 @@ int SrsKafkaBytes::encode(SrsBuffer* buf)
         srs_error("kafka encode bytes data failed. ret=%d", ret);
         return ret;
     }
-    buf->write_bytes(data, _size);
+    buf->write_bytes(_data, _size);
     return ret;
 }
@@ -255,9 +280,9 @@ int SrsKafkaBytes::decode(SrsBuffer* buf)
         return ret;
     }
-    srs_freep(data);
-    data = new char[_size];
-    buf->read_bytes(data, _size);
+    srs_freep(_data);
+    _data = new char[_size];
+    buf->read_bytes(_data, _size);
     return ret;
 }
@@ -523,8 +548,13 @@ int SrsKafkaRawMessage::create(SrsJsonObject* obj)
     // dumps the json to string.
     value->set_value(obj->dumps());
-    // TODO: FIXME: implements it.
-    crc = 0;
+    // crc32 message.
+    crc = srs_crc32_ieee(&magic_byte, 1);
+    crc = srs_crc32_ieee(&attributes, 1, crc);
+    crc = key->crc32(crc);
+    crc = value->crc32(crc);
+    
+    srs_info("crc32 message is %#x", crc);
     message_size = raw_message_size();
@@ -1143,13 +1173,17 @@ SrsKafkaProducerRequest::~SrsKafkaProducerRequest()
 int SrsKafkaProducerRequest::nb_bytes()
 {
-    return 2 + 4 + topics.nb_bytes();
+    return SrsKafkaRequest::nb_bytes() + 2 + 4 + topics.nb_bytes();
 }
 int SrsKafkaProducerRequest::encode(SrsBuffer* buf)
 {
     int ret = ERROR_SUCCESS;
+    if ((ret = SrsKafkaRequest::encode(buf)) != ERROR_SUCCESS) {
+        return ret;
+    }
+    
     if (!buf->require(2 + 4)) {
         ret = ERROR_KAFKA_CODEC_PRODUCER;
         srs_error("kafka encode producer failed. ret=%d", ret);
@@ -1169,6 +1203,10 @@ int SrsKafkaProducerRequest::decode(SrsBuffer* buf)
 {
     int ret = ERROR_SUCCESS;
+    if ((ret = SrsKafkaRequest::decode(buf)) != ERROR_SUCCESS) {
+        return ret;
+    }
+    
     if (!buf->require(2 + 4)) {
         ret = ERROR_KAFKA_CODEC_PRODUCER;
         srs_error("kafka decode producer failed. ret=%d", ret);
@@ -1445,7 +1483,12 @@ int SrsKafkaClient::write_messages(std::string topic, int32_t partition, vector<
     partitions->message_set_size = partitions->messages.nb_bytes();
-    // TODO: FIXME: implements it.
+    // write to kafka cluster.
+    if ((ret = protocol->send_and_free_message(req)) != ERROR_SUCCESS) {
+        srs_error("kafka write producer message failed. ret=%d", ret);
+        return ret;
+    }
+    
     return ret;
 }
--- a/trunk/src/protocol/srs_kafka_stack.hpp
查看文件 @9a47390
+++ b/trunk/src/protocol/srs_kafka_stack.hpp
查看文件 @9a47390
@@ -96,16 +96,19 @@ class SrsKafkaBytes : public ISrsCodec
 {
 private:
     int32_t _size;
-    char* data;
+    char* _data;
 public:
     SrsKafkaBytes();
     SrsKafkaBytes(const char* v, int nb_v);
     virtual ~SrsKafkaBytes();
 public:
+    virtual char* data();
+    virtual int size();
     virtual bool null();
     virtual bool empty();
     virtual void set_value(std::string v);
     virtual void set_value(const char* v, int nb_v);
+    virtual u_int32_t crc32(u_int32_t previous);
 // interface ISrsCodec
 public:
     virtual int nb_bytes();