winlin

merge from srs2

@@ -250,7 +250,7 @@ Compare SRS with other media server. @@ -250,7 +250,7 @@ Compare SRS with other media server.
250 | RTMP Edge | Stable | X | X | Stable | X | 250 | RTMP Edge | Stable | X | X | Stable | X |
251 | RTMP Backup | Stable | X | X | X | X | 251 | RTMP Backup | Stable | X | X | X | X |
252 | VHOST | Stable | X | X | Stable | Stable | 252 | VHOST | Stable | X | X | Stable | Stable |
253 -| Reload | Stable | Stable | X | X | X | 253 +| Reload | Stable | X | X | X | X |
254 | Forward | Stable | X | X | X | X | 254 | Forward | Stable | X | X | X | X |
255 | ATC | Stable | X | X | X | X | 255 | ATC | Stable | X | X | X | X |
256 256
@@ -312,6 +312,7 @@ Remark: @@ -312,6 +312,7 @@ Remark:
312 1. HLS aonly: The HLS audio only streaming delivery. 312 1. HLS aonly: The HLS audio only streaming delivery.
313 1. BW check: The bandwidth check. 313 1. BW check: The bandwidth check.
314 1. Security: To allow or deny stream publish or play. 314 1. Security: To allow or deny stream publish or play.
  315 +1. Reload: Nginx supports reload, but not nginx-rtmp.
315 316
316 ## Releases 317 ## Releases
317 318
@@ -346,6 +347,8 @@ Remark: @@ -346,6 +347,8 @@ Remark:
346 * v3.0, 2015-03-15, fork srs2 and start srs3. 3.0.0 347 * v3.0, 2015-03-15, fork srs2 and start srs3. 3.0.0
347 348
348 ### SRS 2.0 history 349 ### SRS 2.0 history
  350 +
  351 +* v2.0, 2015-05-30, fix [#209](https://github.com/simple-rtmp-server/srs/issues/209) cleanup hls when stop and timeout. 2.0.173.
349 * v2.0, 2015-05-29, fix [#409](https://github.com/simple-rtmp-server/srs/issues/409) support pure video hls. 2.0.172. 352 * v2.0, 2015-05-29, fix [#409](https://github.com/simple-rtmp-server/srs/issues/409) support pure video hls. 2.0.172.
350 * v2.0, 2015-05-28, support [srs-dolphin][srs-dolphin], the multiple-process SRS. 353 * v2.0, 2015-05-28, support [srs-dolphin][srs-dolphin], the multiple-process SRS.
351 * v2.0, 2015-05-24, fix [#404](https://github.com/simple-rtmp-server/srs/issues/404) register handler then start http thread. 2.0.167. 354 * v2.0, 2015-05-24, fix [#404](https://github.com/simple-rtmp-server/srs/issues/404) register handler then start http thread. 2.0.167.
@@ -618,9 +618,14 @@ vhost with-hls.srs.com { @@ -618,9 +618,14 @@ vhost with-hls.srs.com {
618 # h264, vn 618 # h264, vn
619 # default: h264 619 # default: h264
620 hls_vcodec h264; 620 hls_vcodec h264;
621 - # whether cleanup the old ts files. 621 + # whether cleanup the old expired ts files.
622 # default: on 622 # default: on
623 hls_cleanup on; 623 hls_cleanup on;
  624 + # the timeout in seconds to dispose the hls,
  625 + # dispose is to remove all hls files, m3u8 and ts files.
  626 + # when timeout or server terminate, dispose hls.
  627 + # default: 300
  628 + hls_dispose 300;
624 # the max size to notify hls, 629 # the max size to notify hls,
625 # to read max bytes from ts of specified cdn network, 630 # to read max bytes from ts of specified cdn network,
626 # @remark only used when on_hls_notify is config. 631 # @remark only used when on_hls_notify is config.
@@ -109,12 +109,12 @@ stop() { @@ -109,12 +109,12 @@ stop() {
109 ok_msg "Stopping SRS(pid ${srs_pid})..." 109 ok_msg "Stopping SRS(pid ${srs_pid})..."
110 110
111 # process exists, try to kill to stop normally 111 # process exists, try to kill to stop normally
112 - for((i=0;i<30;i++)); do 112 + for((i=0;i<100;i++)); do
113 load_process_info 113 load_process_info
114 if [[ 0 -eq $? ]]; then 114 if [[ 0 -eq $? ]]; then
115 kill -s SIGTERM ${srs_pid} 2>/dev/null 115 kill -s SIGTERM ${srs_pid} 2>/dev/null
116 ret=$?; if [[ 0 -ne $ret ]]; then failed_msg "send signal SIGTERM failed ret=$ret"; return $ret; fi 116 ret=$?; if [[ 0 -ne $ret ]]; then failed_msg "send signal SIGTERM failed ret=$ret"; return $ret; fi
117 - sleep 0.1 117 + sleep 0.3
118 else 118 else
119 ok_msg "SRS stopped by SIGTERM" 119 ok_msg "SRS stopped by SIGTERM"
120 # delete the pid file when stop success. 120 # delete the pid file when stop success.
@@ -1564,7 +1564,8 @@ int SrsConfig::check_config() @@ -1564,7 +1564,8 @@ int SrsConfig::check_config()
1564 string m = conf->at(j)->name.c_str(); 1564 string m = conf->at(j)->name.c_str();
1565 if (m != "enabled" && m != "hls_entry_prefix" && m != "hls_path" && m != "hls_fragment" && m != "hls_window" && m != "hls_on_error" 1565 if (m != "enabled" && m != "hls_entry_prefix" && m != "hls_path" && m != "hls_fragment" && m != "hls_window" && m != "hls_on_error"
1566 && m != "hls_storage" && m != "hls_mount" && m != "hls_td_ratio" && m != "hls_aof_ratio" && m != "hls_acodec" && m != "hls_vcodec" 1566 && m != "hls_storage" && m != "hls_mount" && m != "hls_td_ratio" && m != "hls_aof_ratio" && m != "hls_acodec" && m != "hls_vcodec"
1567 - && m != "hls_m3u8_file" && m != "hls_ts_file" && m != "hls_ts_floor" && m != "hls_cleanup" && m != "hls_nb_notify" && m != "hls_wait_keyframe" 1567 + && m != "hls_m3u8_file" && m != "hls_ts_file" && m != "hls_ts_floor" && m != "hls_cleanup" && m != "hls_nb_notify"
  1568 + && m != "hls_wait_keyframe" && m != "hls_dispose"
1568 ) { 1569 ) {
1569 ret = ERROR_SYSTEM_CONFIG_INVALID; 1570 ret = ERROR_SYSTEM_CONFIG_INVALID;
1570 srs_error("unsupported vhost hls directive %s, ret=%d", m.c_str(), ret); 1571 srs_error("unsupported vhost hls directive %s, ret=%d", m.c_str(), ret);
@@ -3561,6 +3562,24 @@ bool SrsConfig::get_hls_cleanup(string vhost) @@ -3561,6 +3562,24 @@ bool SrsConfig::get_hls_cleanup(string vhost)
3561 return SRS_CONF_PERFER_TRUE(conf->arg0()); 3562 return SRS_CONF_PERFER_TRUE(conf->arg0());
3562 } 3563 }
3563 3564
  3565 +int SrsConfig::get_hls_dispose(string vhost)
  3566 +{
  3567 + SrsConfDirective* conf = get_hls(vhost);
  3568 +
  3569 + int DEFAULT = 300;
  3570 +
  3571 + if (!conf) {
  3572 + return DEFAULT;
  3573 + }
  3574 +
  3575 + conf = conf->get("hls_dispose");
  3576 + if (!conf || conf->arg0().empty()) {
  3577 + return DEFAULT;
  3578 + }
  3579 +
  3580 + return ::atoi(conf->arg0().c_str());
  3581 +}
  3582 +
3564 bool SrsConfig::get_hls_wait_keyframe(string vhost) 3583 bool SrsConfig::get_hls_wait_keyframe(string vhost)
3565 { 3584 {
3566 SrsConfDirective* hls = get_hls(vhost); 3585 SrsConfDirective* hls = get_hls(vhost);
@@ -985,6 +985,10 @@ public: @@ -985,6 +985,10 @@ public:
985 */ 985 */
986 virtual bool get_hls_cleanup(std::string vhost); 986 virtual bool get_hls_cleanup(std::string vhost);
987 /** 987 /**
  988 + * the timeout to dispose the hls.
  989 + */
  990 + virtual int get_hls_dispose(std::string vhost);
  991 + /**
988 * whether reap the ts when got keyframe. 992 * whether reap the ts when got keyframe.
989 */ 993 */
990 virtual bool get_hls_wait_keyframe(std::string vhost); 994 virtual bool get_hls_wait_keyframe(std::string vhost);
@@ -35,6 +35,7 @@ using namespace std; @@ -35,6 +35,7 @@ using namespace std;
35 #include <srs_kernel_error.hpp> 35 #include <srs_kernel_error.hpp>
36 #include <srs_kernel_log.hpp> 36 #include <srs_kernel_log.hpp>
37 #include <srs_app_config.hpp> 37 #include <srs_app_config.hpp>
  38 +#include <srs_app_utility.hpp>
38 39
39 #ifdef SRS_AUTO_FFMPEG_STUB 40 #ifdef SRS_AUTO_FFMPEG_STUB
40 41
@@ -51,6 +52,7 @@ using namespace std; @@ -51,6 +52,7 @@ using namespace std;
51 SrsFFMPEG::SrsFFMPEG(std::string ffmpeg_bin) 52 SrsFFMPEG::SrsFFMPEG(std::string ffmpeg_bin)
52 { 53 {
53 started = false; 54 started = false;
  55 + fast_stopped = false;
54 pid = -1; 56 pid = -1;
55 ffmpeg = ffmpeg_bin; 57 ffmpeg = ffmpeg_bin;
56 58
@@ -402,6 +404,10 @@ int SrsFFMPEG::start() @@ -402,6 +404,10 @@ int SrsFFMPEG::start()
402 404
403 // child process: ffmpeg encoder engine. 405 // child process: ffmpeg encoder engine.
404 if (pid == 0) { 406 if (pid == 0) {
  407 + // ignore the SIGINT and SIGTERM
  408 + signal(SIGINT, SIG_IGN);
  409 + signal(SIGTERM, SIG_IGN);
  410 +
405 // redirect logs to file. 411 // redirect logs to file.
406 int log_fd = -1; 412 int log_fd = -1;
407 int flags = O_CREAT|O_WRONLY|O_APPEND; 413 int flags = O_CREAT|O_WRONLY|O_APPEND;
@@ -479,6 +485,11 @@ int SrsFFMPEG::cycle() @@ -479,6 +485,11 @@ int SrsFFMPEG::cycle()
479 return ret; 485 return ret;
480 } 486 }
481 487
  488 + // ffmpeg is prepare to stop, donot cycle.
  489 + if (fast_stopped) {
  490 + return ret;
  491 + }
  492 +
482 int status = 0; 493 int status = 0;
483 pid_t p = waitpid(pid, &status, WNOHANG); 494 pid_t p = waitpid(pid, &status, WNOHANG);
484 495
@@ -509,24 +520,35 @@ void SrsFFMPEG::stop() @@ -509,24 +520,35 @@ void SrsFFMPEG::stop()
509 // when rewind, upstream will stop publish(unpublish), 520 // when rewind, upstream will stop publish(unpublish),
510 // unpublish event will stop all ffmpeg encoders, 521 // unpublish event will stop all ffmpeg encoders,
511 // then publish will start all ffmpeg encoders. 522 // then publish will start all ffmpeg encoders.
512 - if (pid > 0) {  
513 - if (kill(pid, SIGKILL) < 0) {  
514 - srs_warn("kill the encoder failed, ignored. pid=%d", pid);  
515 - }  
516 -  
517 - // wait for the ffmpeg to quit.  
518 - // ffmpeg will gracefully quit if signal is:  
519 - // 1) SIGHUP 2) SIGINT 3) SIGQUIT  
520 - // other signals, directly exit(123), for example:  
521 - // 9) SIGKILL 15) SIGTERM  
522 - int status = 0;  
523 - if (waitpid(pid, &status, 0) < 0) {  
524 - srs_warn("wait the encoder quit failed, ignored. pid=%d", pid);  
525 - }  
526 -  
527 - srs_trace("stop the encoder success. pid=%d", pid);  
528 - pid = -1; 523 + int ret = srs_kill_forced(pid);
  524 + if (ret != ERROR_SUCCESS) {
  525 + srs_warn("ignore kill the encoder failed, pid=%d. ret=%d", pid, ret);
  526 + return;
529 } 527 }
  528 +
  529 + // terminated, set started to false to stop the cycle.
  530 + started = false;
  531 +}
  532 +
  533 +void SrsFFMPEG::fast_stop()
  534 +{
  535 + int ret = ERROR_SUCCESS;
  536 +
  537 + if (!started) {
  538 + return;
  539 + }
  540 +
  541 + if (pid <= 0) {
  542 + return;
  543 + }
  544 +
  545 + if (kill(pid, SIGTERM) < 0) {
  546 + ret = ERROR_SYSTEM_KILL;
  547 + srs_warn("ignore fast stop ffmpeg failed, pid=%d. ret=%d", pid, ret);
  548 + return;
  549 + }
  550 +
  551 + return;
530 } 552 }
531 553
532 #endif 554 #endif
@@ -45,6 +45,8 @@ class SrsFFMPEG @@ -45,6 +45,8 @@ class SrsFFMPEG
45 { 45 {
46 private: 46 private:
47 bool started; 47 bool started;
  48 + // whether SIGINT send but need to wait or SIGKILL.
  49 + bool fast_stopped;
48 pid_t pid; 50 pid_t pid;
49 private: 51 private:
50 std::string log_file; 52 std::string log_file;
@@ -83,7 +85,25 @@ public: @@ -83,7 +85,25 @@ public:
83 virtual int initialize_copy(); 85 virtual int initialize_copy();
84 virtual int start(); 86 virtual int start();
85 virtual int cycle(); 87 virtual int cycle();
  88 + /**
  89 + * send SIGTERM then SIGKILL to ensure the process stopped.
  90 + * the stop will wait [0, SRS_PROCESS_QUIT_TIMEOUT_MS] depends on the
  91 + * process quit timeout.
  92 + * @remark use fast_stop before stop one by one, when got lots of process to quit.
  93 + */
86 virtual void stop(); 94 virtual void stop();
  95 +public:
  96 + /**
  97 + * the fast stop is to send a SIGTERM.
  98 + * for example, the ingesters owner lots of FFMPEG, it will take a long time
  99 + * to stop one by one, instead the ingesters can fast_stop all FFMPEG, then
  100 + * wait one by one to stop, it's more faster.
  101 + * @remark user must use stop() to ensure the ffmpeg to stopped.
  102 + * @remark we got N processes to stop, compare the time we spend,
  103 + * when use stop without fast_stop, we spend maybe [0, SRS_PROCESS_QUIT_TIMEOUT_MS * N]
  104 + * but use fast_stop then stop, the time is almost [0, SRS_PROCESS_QUIT_TIMEOUT_MS].
  105 + */
  106 + virtual void fast_stop();
87 }; 107 };
88 108
89 #endif 109 #endif
@@ -305,6 +305,37 @@ SrsHlsMuxer::~SrsHlsMuxer() @@ -305,6 +305,37 @@ SrsHlsMuxer::~SrsHlsMuxer()
305 srs_freep(context); 305 srs_freep(context);
306 } 306 }
307 307
  308 +void SrsHlsMuxer::dispose()
  309 +{
  310 + if (should_write_file) {
  311 + std::vector<SrsHlsSegment*>::iterator it;
  312 + for (it = segments.begin(); it != segments.end(); ++it) {
  313 + SrsHlsSegment* segment = *it;
  314 + if (unlink(segment->full_path.c_str()) < 0) {
  315 + srs_warn("dispose unlink path failed, file=%s.", segment->full_path.c_str());
  316 + }
  317 + srs_freep(segment);
  318 + }
  319 + segments.clear();
  320 +
  321 + if (current) {
  322 + std::string path = current->full_path + ".tmp";
  323 + if (unlink(path.c_str()) < 0) {
  324 + srs_warn("dispose unlink path failed, file=%s", path.c_str());
  325 + }
  326 + srs_freep(current);
  327 + }
  328 +
  329 + if (unlink(m3u8.c_str()) < 0) {
  330 + srs_warn("dispose unlink path failed. file=%s", m3u8.c_str());
  331 + }
  332 + }
  333 +
  334 + // TODO: FIXME: support hls dispose in HTTP cache.
  335 +
  336 + srs_trace("gracefully dispose hls %s", req? req->get_stream_url().c_str() : "");
  337 +}
  338 +
308 int SrsHlsMuxer::sequence_no() 339 int SrsHlsMuxer::sequence_no()
309 { 340 {
310 return _sequence_no; 341 return _sequence_no;
@@ -720,6 +751,9 @@ int SrsHlsMuxer::segment_close(string log_desc) @@ -720,6 +751,9 @@ int SrsHlsMuxer::segment_close(string log_desc)
720 std::string tmp_file = current->full_path + ".tmp"; 751 std::string tmp_file = current->full_path + ".tmp";
721 if (should_write_file) { 752 if (should_write_file) {
722 unlink(tmp_file.c_str()); 753 unlink(tmp_file.c_str());
  754 + if (unlink(tmp_file.c_str()) < 0) {
  755 + srs_warn("drop unlink path failed, file=%s.", tmp_file.c_str());
  756 + }
723 } 757 }
724 758
725 srs_freep(current); 759 srs_freep(current);
@@ -754,7 +788,9 @@ int SrsHlsMuxer::segment_close(string log_desc) @@ -754,7 +788,9 @@ int SrsHlsMuxer::segment_close(string log_desc)
754 SrsHlsSegment* segment = segment_to_remove[i]; 788 SrsHlsSegment* segment = segment_to_remove[i];
755 789
756 if (hls_cleanup) { 790 if (hls_cleanup) {
757 - unlink(segment->full_path.c_str()); 791 + if (unlink(segment->full_path.c_str()) < 0) {
  792 + srs_warn("cleanup unlink path failed, file=%s.", segment->full_path.c_str());
  793 + }
758 } 794 }
759 795
760 srs_freep(segment); 796 srs_freep(segment);
@@ -1085,6 +1121,8 @@ SrsHls::SrsHls() @@ -1085,6 +1121,8 @@ SrsHls::SrsHls()
1085 handler = NULL; 1121 handler = NULL;
1086 1122
1087 hls_enabled = false; 1123 hls_enabled = false;
  1124 + hls_can_dispose = false;
  1125 + last_update_time = 0;
1088 1126
1089 codec = new SrsAvcAacCodec(); 1127 codec = new SrsAvcAacCodec();
1090 sample = new SrsCodecSample(); 1128 sample = new SrsCodecSample();
@@ -1109,6 +1147,46 @@ SrsHls::~SrsHls() @@ -1109,6 +1147,46 @@ SrsHls::~SrsHls()
1109 srs_freep(pprint); 1147 srs_freep(pprint);
1110 } 1148 }
1111 1149
  1150 +void SrsHls::dispose()
  1151 +{
  1152 + if (hls_enabled) {
  1153 + on_unpublish();
  1154 + }
  1155 +
  1156 + muxer->dispose();
  1157 +}
  1158 +
  1159 +int SrsHls::cycle()
  1160 +{
  1161 + int ret = ERROR_SUCCESS;
  1162 +
  1163 + srs_info("hls cycle for source %d", source->source_id());
  1164 +
  1165 + if (last_update_time <= 0) {
  1166 + last_update_time = srs_get_system_time_ms();
  1167 + }
  1168 +
  1169 + if (!_req) {
  1170 + return ret;
  1171 + }
  1172 +
  1173 + int hls_dispose = _srs_config->get_hls_dispose(_req->vhost) * 1000;
  1174 + if (srs_get_system_time_ms() - last_update_time <= hls_dispose) {
  1175 + return ret;
  1176 + }
  1177 + last_update_time = srs_get_system_time_ms();
  1178 +
  1179 + if (!hls_can_dispose) {
  1180 + return ret;
  1181 + }
  1182 + hls_can_dispose = false;
  1183 +
  1184 + srs_trace("hls cycle to dispose hls %s, timeout=%dms", _req->get_stream_url().c_str(), hls_dispose);
  1185 + dispose();
  1186 +
  1187 + return ret;
  1188 +}
  1189 +
1112 int SrsHls::initialize(SrsSource* s, ISrsHlsHandler* h) 1190 int SrsHls::initialize(SrsSource* s, ISrsHlsHandler* h)
1113 { 1191 {
1114 int ret = ERROR_SUCCESS; 1192 int ret = ERROR_SUCCESS;
@@ -1127,6 +1205,11 @@ int SrsHls::on_publish(SrsRequest* req) @@ -1127,6 +1205,11 @@ int SrsHls::on_publish(SrsRequest* req)
1127 { 1205 {
1128 int ret = ERROR_SUCCESS; 1206 int ret = ERROR_SUCCESS;
1129 1207
  1208 + _req = req;
  1209 +
  1210 + // update the hls time, for hls_dispose.
  1211 + last_update_time = srs_get_system_time_ms();
  1212 +
1130 // support multiple publish. 1213 // support multiple publish.
1131 if (hls_enabled) { 1214 if (hls_enabled) {
1132 return ret; 1215 return ret;
@@ -1144,6 +1227,9 @@ int SrsHls::on_publish(SrsRequest* req) @@ -1144,6 +1227,9 @@ int SrsHls::on_publish(SrsRequest* req)
1144 // if enabled, open the muxer. 1227 // if enabled, open the muxer.
1145 hls_enabled = true; 1228 hls_enabled = true;
1146 1229
  1230 + // ok, the hls can be dispose, or need to be dispose.
  1231 + hls_can_dispose = true;
  1232 +
1147 // notice the source to get the cached sequence header. 1233 // notice the source to get the cached sequence header.
1148 // when reload to start hls, hls will never get the sequence header in stream, 1234 // when reload to start hls, hls will never get the sequence header in stream,
1149 // use the SrsSource.on_hls_start to push the sequence header to HLS. 1235 // use the SrsSource.on_hls_start to push the sequence header to HLS.
@@ -1195,6 +1281,9 @@ int SrsHls::on_audio(SrsSharedPtrMessage* shared_audio) @@ -1195,6 +1281,9 @@ int SrsHls::on_audio(SrsSharedPtrMessage* shared_audio)
1195 if (!hls_enabled) { 1281 if (!hls_enabled) {
1196 return ret; 1282 return ret;
1197 } 1283 }
  1284 +
  1285 + // update the hls time, for hls_dispose.
  1286 + last_update_time = srs_get_system_time_ms();
1198 1287
1199 SrsSharedPtrMessage* audio = shared_audio->copy(); 1288 SrsSharedPtrMessage* audio = shared_audio->copy();
1200 SrsAutoFree(SrsSharedPtrMessage, audio); 1289 SrsAutoFree(SrsSharedPtrMessage, audio);
@@ -1256,6 +1345,9 @@ int SrsHls::on_video(SrsSharedPtrMessage* shared_video) @@ -1256,6 +1345,9 @@ int SrsHls::on_video(SrsSharedPtrMessage* shared_video)
1256 if (!hls_enabled) { 1345 if (!hls_enabled) {
1257 return ret; 1346 return ret;
1258 } 1347 }
  1348 +
  1349 + // update the hls time, for hls_dispose.
  1350 + last_update_time = srs_get_system_time_ms();
1259 1351
1260 SrsSharedPtrMessage* video = shared_video->copy(); 1352 SrsSharedPtrMessage* video = shared_video->copy();
1261 SrsAutoFree(SrsSharedPtrMessage, video); 1353 SrsAutoFree(SrsSharedPtrMessage, video);
@@ -260,6 +260,8 @@ public: @@ -260,6 +260,8 @@ public:
260 SrsHlsMuxer(); 260 SrsHlsMuxer();
261 virtual ~SrsHlsMuxer(); 261 virtual ~SrsHlsMuxer();
262 public: 262 public:
  263 + virtual void dispose();
  264 +public:
263 virtual int sequence_no(); 265 virtual int sequence_no();
264 virtual std::string ts_url(); 266 virtual std::string ts_url();
265 virtual double duration(); 267 virtual double duration();
@@ -379,7 +381,11 @@ private: @@ -379,7 +381,11 @@ private:
379 SrsHlsCache* hls_cache; 381 SrsHlsCache* hls_cache;
380 ISrsHlsHandler* handler; 382 ISrsHlsHandler* handler;
381 private: 383 private:
  384 + SrsRequest* _req;
382 bool hls_enabled; 385 bool hls_enabled;
  386 + bool hls_can_dispose;
  387 + int64_t last_update_time;
  388 +private:
383 SrsSource* source; 389 SrsSource* source;
384 SrsAvcAacCodec* codec; 390 SrsAvcAacCodec* codec;
385 SrsCodecSample* sample; 391 SrsCodecSample* sample;
@@ -403,6 +409,9 @@ public: @@ -403,6 +409,9 @@ public:
403 SrsHls(); 409 SrsHls();
404 virtual ~SrsHls(); 410 virtual ~SrsHls();
405 public: 411 public:
  412 + virtual void dispose();
  413 + virtual int cycle();
  414 +public:
406 /** 415 /**
407 * initialize the hls by handler and source. 416 * initialize the hls by handler and source.
408 */ 417 */
@@ -51,6 +51,11 @@ SrsIngesterFFMPEG::~SrsIngesterFFMPEG() @@ -51,6 +51,11 @@ SrsIngesterFFMPEG::~SrsIngesterFFMPEG()
51 srs_freep(ffmpeg); 51 srs_freep(ffmpeg);
52 } 52 }
53 53
  54 +void SrsIngesterFFMPEG::fast_stop()
  55 +{
  56 + ffmpeg->fast_stop();
  57 +}
  58 +
54 SrsIngester::SrsIngester() 59 SrsIngester::SrsIngester()
55 { 60 {
56 _srs_config->subscribe(this); 61 _srs_config->subscribe(this);
@@ -159,6 +164,23 @@ int SrsIngester::parse_engines(SrsConfDirective* vhost, SrsConfDirective* ingest @@ -159,6 +164,23 @@ int SrsIngester::parse_engines(SrsConfDirective* vhost, SrsConfDirective* ingest
159 return ret; 164 return ret;
160 } 165 }
161 166
  167 +void SrsIngester::dispose()
  168 +{
  169 + // first, use fast stop to notice all FFMPEG to quit gracefully.
  170 + std::vector<SrsIngesterFFMPEG*>::iterator it;
  171 + for (it = ingesters.begin(); it != ingesters.end(); ++it) {
  172 + SrsIngesterFFMPEG* ingester = *it;
  173 + ingester->fast_stop();
  174 + }
  175 +
  176 + if (!ingesters.empty()) {
  177 + srs_trace("fast stop all ingesters ok.");
  178 + }
  179 +
  180 + // then, use stop to wait FFMPEG quit one by one and send SIGKILL if needed.
  181 + stop();
  182 +}
  183 +
162 void SrsIngester::stop() 184 void SrsIngester::stop()
163 { 185 {
164 pthread->stop(); 186 pthread->stop();
@@ -52,6 +52,9 @@ public: @@ -52,6 +52,9 @@ public:
52 52
53 SrsIngesterFFMPEG(SrsFFMPEG* _ffmpeg, std::string _vhost, std::string _id); 53 SrsIngesterFFMPEG(SrsFFMPEG* _ffmpeg, std::string _vhost, std::string _id);
54 virtual ~SrsIngesterFFMPEG(); 54 virtual ~SrsIngesterFFMPEG();
  55 +
  56 + // @see SrsFFMPEG.fast_stop().
  57 + virtual void fast_stop();
55 }; 58 };
56 59
57 /** 60 /**
@@ -70,6 +73,8 @@ public: @@ -70,6 +73,8 @@ public:
70 SrsIngester(); 73 SrsIngester();
71 virtual ~SrsIngester(); 74 virtual ~SrsIngester();
72 public: 75 public:
  76 + virtual void dispose();
  77 +public:
73 virtual int start(); 78 virtual int start();
74 virtual void stop(); 79 virtual void stop();
75 // interface ISrsReusableThreadHandler. 80 // interface ISrsReusableThreadHandler.
@@ -480,6 +480,7 @@ SrsServer::SrsServer() @@ -480,6 +480,7 @@ SrsServer::SrsServer()
480 { 480 {
481 signal_reload = false; 481 signal_reload = false;
482 signal_gmc_stop = false; 482 signal_gmc_stop = false;
  483 + signal_gracefully_quit = false;
483 pid_fd = -1; 484 pid_fd = -1;
484 485
485 signal_manager = NULL; 486 signal_manager = NULL;
@@ -519,7 +520,7 @@ void SrsServer::destroy() @@ -519,7 +520,7 @@ void SrsServer::destroy()
519 close_listeners(SrsListenerHttpStream); 520 close_listeners(SrsListenerHttpStream);
520 521
521 #ifdef SRS_AUTO_INGEST 522 #ifdef SRS_AUTO_INGEST
522 - ingester->stop(); 523 + ingester->dispose();
523 #endif 524 #endif
524 525
525 #ifdef SRS_AUTO_HTTP_API 526 #ifdef SRS_AUTO_HTTP_API
@@ -555,6 +556,21 @@ void SrsServer::destroy() @@ -555,6 +556,21 @@ void SrsServer::destroy()
555 // and segment fault. 556 // and segment fault.
556 } 557 }
557 558
  559 +void SrsServer::dispose()
  560 +{
  561 + _srs_config->unsubscribe(this);
  562 +
  563 +#ifdef SRS_AUTO_INGEST
  564 + ingester->dispose();
  565 + srs_trace("gracefully dispose ingesters");
  566 +#endif
  567 +
  568 + SrsSource::dispose_all();
  569 + srs_trace("gracefully dispose sources");
  570 +
  571 + srs_trace("terminate server");
  572 +}
  573 +
558 int SrsServer::initialize(ISrsServerCycle* cycle_handler) 574 int SrsServer::initialize(ISrsServerCycle* cycle_handler)
559 { 575 {
560 int ret = ERROR_SUCCESS; 576 int ret = ERROR_SUCCESS;
@@ -831,6 +847,7 @@ int SrsServer::cycle() @@ -831,6 +847,7 @@ int SrsServer::cycle()
831 srs_warn("system quit"); 847 srs_warn("system quit");
832 #else 848 #else
833 srs_warn("main cycle terminated, system quit normally."); 849 srs_warn("main cycle terminated, system quit normally.");
  850 + dispose();
834 exit(0); 851 exit(0);
835 #endif 852 #endif
836 853
@@ -877,9 +894,9 @@ void SrsServer::on_signal(int signo) @@ -877,9 +894,9 @@ void SrsServer::on_signal(int signo)
877 return; 894 return;
878 } 895 }
879 896
880 - if (signo == SIGTERM) {  
881 - srs_trace("user terminate program");  
882 - exit(0); 897 + if (signo == SIGTERM && !signal_gracefully_quit) {
  898 + srs_trace("user terminate program, gracefully quit.");
  899 + signal_gracefully_quit = true;
883 return; 900 return;
884 } 901 }
885 } 902 }
@@ -903,7 +920,7 @@ int SrsServer::do_cycle() @@ -903,7 +920,7 @@ int SrsServer::do_cycle()
903 920
904 // the deamon thread, update the time cache 921 // the deamon thread, update the time cache
905 while (true) { 922 while (true) {
906 - if(handler && (ret = handler->on_cycle(conns.size())) != ERROR_SUCCESS){ 923 + if(handler && (ret = handler->on_cycle((int)conns.size())) != ERROR_SUCCESS){
907 srs_error("cycle handle failed. ret=%d", ret); 924 srs_error("cycle handle failed. ret=%d", ret);
908 return ret; 925 return ret;
909 } 926 }
@@ -917,12 +934,18 @@ int SrsServer::do_cycle() @@ -917,12 +934,18 @@ int SrsServer::do_cycle()
917 934
918 for (int i = 0; i < temp_max; i++) { 935 for (int i = 0; i < temp_max; i++) {
919 st_usleep(SRS_SYS_CYCLE_INTERVAL * 1000); 936 st_usleep(SRS_SYS_CYCLE_INTERVAL * 1000);
  937 +
  938 + // gracefully quit for SIGINT or SIGTERM.
  939 + if (signal_gracefully_quit) {
  940 + srs_trace("cleanup for gracefully terminate.");
  941 + return ret;
  942 + }
920 943
921 -// for gperf heap checker,  
922 -// @see: research/gperftools/heap-checker/heap_checker.cc  
923 -// if user interrupt the program, exit to check mem leak.  
924 -// but, if gperf, use reload to ensure main return normally,  
925 -// because directly exit will cause core-dump. 944 + // for gperf heap checker,
  945 + // @see: research/gperftools/heap-checker/heap_checker.cc
  946 + // if user interrupt the program, exit to check mem leak.
  947 + // but, if gperf, use reload to ensure main return normally,
  948 + // because directly exit will cause core-dump.
926 #ifdef SRS_AUTO_GPERF_MC 949 #ifdef SRS_AUTO_GPERF_MC
927 if (signal_gmc_stop) { 950 if (signal_gmc_stop) {
928 srs_warn("gmc got singal to stop server."); 951 srs_warn("gmc got singal to stop server.");
@@ -930,6 +953,7 @@ int SrsServer::do_cycle() @@ -930,6 +953,7 @@ int SrsServer::do_cycle()
930 } 953 }
931 #endif 954 #endif
932 955
  956 + // do reload the config.
933 if (signal_reload) { 957 if (signal_reload) {
934 signal_reload = false; 958 signal_reload = false;
935 srs_info("get signal reload, to reload the config."); 959 srs_info("get signal reload, to reload the config.");
@@ -941,6 +965,11 @@ int SrsServer::do_cycle() @@ -941,6 +965,11 @@ int SrsServer::do_cycle()
941 srs_trace("reload config success."); 965 srs_trace("reload config success.");
942 } 966 }
943 967
  968 + // notice the stream sources to cycle.
  969 + if ((ret = SrsSource::cycle_all()) != ERROR_SUCCESS) {
  970 + return ret;
  971 + }
  972 +
944 // update the cache time 973 // update the cache time
945 if ((i % SRS_SYS_TIME_RESOLUTION_MS_TIMES) == 0) { 974 if ((i % SRS_SYS_TIME_RESOLUTION_MS_TIMES) == 0) {
946 srs_info("update current time cache."); 975 srs_info("update current time cache.");
@@ -275,16 +275,22 @@ private: @@ -275,16 +275,22 @@ private:
275 */ 275 */
276 bool signal_reload; 276 bool signal_reload;
277 bool signal_gmc_stop; 277 bool signal_gmc_stop;
  278 + bool signal_gracefully_quit;
278 public: 279 public:
279 SrsServer(); 280 SrsServer();
280 virtual ~SrsServer(); 281 virtual ~SrsServer();
281 -public: 282 +private:
282 /** 283 /**
283 * the destroy is for gmc to analysis the memory leak, 284 * the destroy is for gmc to analysis the memory leak,
284 * if not destroy global/static data, the gmc will warning memory leak. 285 * if not destroy global/static data, the gmc will warning memory leak.
285 * in service, server never destroy, directly exit when restart. 286 * in service, server never destroy, directly exit when restart.
286 */ 287 */
287 virtual void destroy(); 288 virtual void destroy();
  289 + /**
  290 + * when SIGTERM, SRS should do cleanup, for example,
  291 + * to stop all ingesters, cleanup HLS and dvr.
  292 + */
  293 + virtual void dispose();
288 // server startup workflow, @see run_master() 294 // server startup workflow, @see run_master()
289 public: 295 public:
290 virtual int initialize(ISrsServerCycle* cycle_handler); 296 virtual int initialize(ISrsServerCycle* cycle_handler);
@@ -771,6 +771,32 @@ SrsSource* SrsSource::fetch(std::string vhost, std::string app, std::string stre @@ -771,6 +771,32 @@ SrsSource* SrsSource::fetch(std::string vhost, std::string app, std::string stre
771 return source; 771 return source;
772 } 772 }
773 773
  774 +void SrsSource::dispose_all()
  775 +{
  776 + std::map<std::string, SrsSource*>::iterator it;
  777 + for (it = pool.begin(); it != pool.end(); ++it) {
  778 + SrsSource* source = it->second;
  779 + source->dispose();
  780 + }
  781 + return;
  782 +}
  783 +
  784 +int SrsSource::cycle_all()
  785 +{
  786 + int ret = ERROR_SUCCESS;
  787 +
  788 + // TODO: FIXME: support remove dead source for a long time.
  789 + std::map<std::string, SrsSource*>::iterator it;
  790 + for (it = pool.begin(); it != pool.end(); ++it) {
  791 + SrsSource* source = it->second;
  792 + if ((ret = source->cycle()) != ERROR_SUCCESS) {
  793 + return ret;
  794 + }
  795 + }
  796 +
  797 + return ret;
  798 +}
  799 +
774 void SrsSource::destroy() 800 void SrsSource::destroy()
775 { 801 {
776 std::map<std::string, SrsSource*>::iterator it; 802 std::map<std::string, SrsSource*>::iterator it;
@@ -909,6 +935,26 @@ SrsSource::~SrsSource() @@ -909,6 +935,26 @@ SrsSource::~SrsSource()
909 srs_freep(_req); 935 srs_freep(_req);
910 } 936 }
911 937
  938 +void SrsSource::dispose()
  939 +{
  940 +#ifdef SRS_AUTO_HLS
  941 + hls->dispose();
  942 +#endif
  943 +}
  944 +
  945 +int SrsSource::cycle()
  946 +{
  947 + int ret = ERROR_SUCCESS;
  948 +
  949 +#ifdef SRS_AUTO_HLS
  950 + if ((ret = hls->cycle()) != ERROR_SUCCESS) {
  951 + return ret;
  952 + }
  953 +#endif
  954 +
  955 + return ret;
  956 +}
  957 +
912 int SrsSource::initialize(SrsRequest* r, ISrsSourceHandler* h, ISrsHlsHandler* hh) 958 int SrsSource::initialize(SrsRequest* r, ISrsSourceHandler* h, ISrsHlsHandler* hh)
913 { 959 {
914 int ret = ERROR_SUCCESS; 960 int ret = ERROR_SUCCESS;
@@ -411,6 +411,11 @@ public: @@ -411,6 +411,11 @@ public:
411 */ 411 */
412 static SrsSource* fetch(std::string vhost, std::string app, std::string stream); 412 static SrsSource* fetch(std::string vhost, std::string app, std::string stream);
413 /** 413 /**
  414 + * dispose and cycle all sources.
  415 + */
  416 + static void dispose_all();
  417 + static int cycle_all();
  418 + /**
414 * when system exit, destroy the sources, 419 * when system exit, destroy the sources,
415 * for gmc to analysis mem leaks. 420 * for gmc to analysis mem leaks.
416 */ 421 */
@@ -486,6 +491,9 @@ private: @@ -486,6 +491,9 @@ private:
486 public: 491 public:
487 SrsSource(); 492 SrsSource();
488 virtual ~SrsSource(); 493 virtual ~SrsSource();
  494 +public:
  495 + virtual void dispose();
  496 + virtual int cycle();
489 // initialize, get and setter. 497 // initialize, get and setter.
490 public: 498 public:
491 /** 499 /**
@@ -27,6 +27,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @@ -27,6 +27,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27 #include <unistd.h> 27 #include <unistd.h>
28 #include <ifaddrs.h> 28 #include <ifaddrs.h>
29 #include <arpa/inet.h> 29 #include <arpa/inet.h>
  30 +#include <signal.h>
30 31
31 #ifdef SRS_OSX 32 #ifdef SRS_OSX
32 #include <sys/sysctl.h> 33 #include <sys/sysctl.h>
@@ -43,6 +44,9 @@ using namespace std; @@ -43,6 +44,9 @@ using namespace std;
43 #include <srs_app_json.hpp> 44 #include <srs_app_json.hpp>
44 #include <srs_kernel_stream.hpp> 45 #include <srs_kernel_stream.hpp>
45 46
  47 +// the longest time to wait for a process to quit.
  48 +#define SRS_PROCESS_QUIT_TIMEOUT_MS 1000
  49 +
46 int srs_socket_connect(string server, int port, int64_t timeout, st_netfd_t* pstfd) 50 int srs_socket_connect(string server, int port, int64_t timeout, st_netfd_t* pstfd)
47 { 51 {
48 int ret = ERROR_SUCCESS; 52 int ret = ERROR_SUCCESS;
@@ -222,6 +226,62 @@ void srs_parse_endpoint(string ip_port, string& ip, int& port) @@ -222,6 +226,62 @@ void srs_parse_endpoint(string ip_port, string& ip, int& port)
222 port = ::atoi(the_port.c_str()); 226 port = ::atoi(the_port.c_str());
223 } 227 }
224 228
  229 +int srs_kill_forced(int& pid)
  230 +{
  231 + int ret = ERROR_SUCCESS;
  232 +
  233 + if (pid <= 0) {
  234 + return ret;
  235 + }
  236 +
  237 + // first, try kill by SIGTERM.
  238 + if (kill(pid, SIGTERM) < 0) {
  239 + return ERROR_SYSTEM_KILL;
  240 + }
  241 +
  242 + // wait to quit.
  243 + srs_trace("send SIGTERM to pid=%d", pid);
  244 + for (int i = 0; i < SRS_PROCESS_QUIT_TIMEOUT_MS / 10; i++) {
  245 + int status = 0;
  246 + pid_t qpid = -1;
  247 + if ((qpid = waitpid(pid, &status, WNOHANG)) < 0) {
  248 + return ERROR_SYSTEM_KILL;
  249 + }
  250 +
  251 + // 0 is not quit yet.
  252 + if (qpid == 0) {
  253 + st_usleep(10 * 1000);
  254 + continue;
  255 + }
  256 +
  257 + // killed, set pid to -1.
  258 + srs_trace("SIGTERM stop process pid=%d ok.", pid);
  259 + pid = -1;
  260 +
  261 + return ret;
  262 + }
  263 +
  264 + // then, try kill by SIGKILL.
  265 + if (kill(pid, SIGKILL) < 0) {
  266 + return ERROR_SYSTEM_KILL;
  267 + }
  268 +
  269 + // wait for the process to quit.
  270 + // for example, ffmpeg will gracefully quit if signal is:
  271 + // 1) SIGHUP 2) SIGINT 3) SIGQUIT
  272 + // other signals, directly exit(123), for example:
  273 + // 9) SIGKILL 15) SIGTERM
  274 + int status = 0;
  275 + if (waitpid(pid, &status, 0) < 0) {
  276 + return ERROR_SYSTEM_KILL;
  277 + }
  278 +
  279 + srs_trace("SIGKILL stop process pid=%d ok.", pid);
  280 + pid = -1;
  281 +
  282 + return ret;
  283 +}
  284 +
225 static SrsRusage _srs_system_rusage; 285 static SrsRusage _srs_system_rusage;
226 286
227 SrsRusage::SrsRusage() 287 SrsRusage::SrsRusage()
@@ -422,7 +482,7 @@ void srs_update_proc_stat() @@ -422,7 +482,7 @@ void srs_update_proc_stat()
422 // @see https://github.com/simple-rtmp-server/srs/issues/397 482 // @see https://github.com/simple-rtmp-server/srs/issues/397
423 static int user_hz = 0; 483 static int user_hz = 0;
424 if (user_hz <= 0) { 484 if (user_hz <= 0) {
425 - user_hz = sysconf(_SC_CLK_TCK); 485 + user_hz = (int)sysconf(_SC_CLK_TCK);
426 srs_trace("USER_HZ=%d", user_hz); 486 srs_trace("USER_HZ=%d", user_hz);
427 srs_assert(user_hz > 0); 487 srs_assert(user_hz > 0);
428 } 488 }
@@ -646,12 +706,12 @@ void srs_update_disk_stat() @@ -646,12 +706,12 @@ void srs_update_disk_stat()
646 706
647 if (o.pgpgin > 0 && r.pgpgin > o.pgpgin && duration_ms > 0) { 707 if (o.pgpgin > 0 && r.pgpgin > o.pgpgin && duration_ms > 0) {
648 // KBps = KB * 1000 / ms = KB/s 708 // KBps = KB * 1000 / ms = KB/s
649 - r.in_KBps = (r.pgpgin - o.pgpgin) * 1000 / duration_ms; 709 + r.in_KBps = (int)((r.pgpgin - o.pgpgin) * 1000 / duration_ms);
650 } 710 }
651 711
652 if (o.pgpgout > 0 && r.pgpgout > o.pgpgout && duration_ms > 0) { 712 if (o.pgpgout > 0 && r.pgpgout > o.pgpgout && duration_ms > 0) {
653 // KBps = KB * 1000 / ms = KB/s 713 // KBps = KB * 1000 / ms = KB/s
654 - r.out_KBps = (r.pgpgout - o.pgpgout) * 1000 / duration_ms; 714 + r.out_KBps = (int)((r.pgpgout - o.pgpgout) * 1000 / duration_ms);
655 } 715 }
656 } 716 }
657 717
@@ -771,8 +831,8 @@ SrsCpuInfo* srs_get_cpuinfo() @@ -771,8 +831,8 @@ SrsCpuInfo* srs_get_cpuinfo()
771 // initialize cpu info. 831 // initialize cpu info.
772 cpu = new SrsCpuInfo(); 832 cpu = new SrsCpuInfo();
773 cpu->ok = true; 833 cpu->ok = true;
774 - cpu->nb_processors = sysconf(_SC_NPROCESSORS_CONF);  
775 - cpu->nb_processors_online = sysconf(_SC_NPROCESSORS_ONLN); 834 + cpu->nb_processors = (int)sysconf(_SC_NPROCESSORS_CONF);
  835 + cpu->nb_processors_online = (int)sysconf(_SC_NPROCESSORS_ONLN);
776 836
777 return cpu; 837 return cpu;
778 } 838 }
@@ -80,6 +80,14 @@ extern std::string srs_path_build_timestamp(std::string template_path); @@ -80,6 +80,14 @@ extern std::string srs_path_build_timestamp(std::string template_path);
80 extern void srs_parse_endpoint(std::string ip_port, std::string& ip, std::string& port); 80 extern void srs_parse_endpoint(std::string ip_port, std::string& ip, std::string& port);
81 extern void srs_parse_endpoint(std::string ip_port, std::string& ip, int& port); 81 extern void srs_parse_endpoint(std::string ip_port, std::string& ip, int& port);
82 82
  83 +/**
  84 + * kill the pid by SIGINT, then wait to quit,
  85 + * kill the pid by SIGKILL again when exceed the timeout.
  86 + * @param pid the pid to kill. ignore for -1. set to -1 when killed.
  87 + * @return an int error code.
  88 + */
  89 +extern int srs_kill_forced(int& pid);
  90 +
83 // current process resouce usage. 91 // current process resouce usage.
84 // @see: man getrusage 92 // @see: man getrusage
85 class SrsRusage 93 class SrsRusage
@@ -96,6 +96,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @@ -96,6 +96,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
96 #define ERROR_SYSTEM_TIME 1055 96 #define ERROR_SYSTEM_TIME 1055
97 #define ERROR_SYSTEM_DIR_EXISTS 1056 97 #define ERROR_SYSTEM_DIR_EXISTS 1056
98 #define ERROR_SYSTEM_CREATE_DIR 1057 98 #define ERROR_SYSTEM_CREATE_DIR 1057
  99 +#define ERROR_SYSTEM_KILL 1058
99 100
100 /////////////////////////////////////////////////////// 101 ///////////////////////////////////////////////////////
101 // RTMP protocol error. 102 // RTMP protocol error.