winlin

for #374, use fast stop for ingesters to stop many FFMPEG.

@@ -109,12 +109,12 @@ stop() { @@ -109,12 +109,12 @@ stop() {
109 ok_msg "Stopping SRS(pid ${srs_pid})..." 109 ok_msg "Stopping SRS(pid ${srs_pid})..."
110 110
111 # process exists, try to kill to stop normally 111 # process exists, try to kill to stop normally
112 - for((i=0;i<30;i++)); do 112 + for((i=0;i<100;i++)); do
113 load_process_info 113 load_process_info
114 if [[ 0 -eq $? ]]; then 114 if [[ 0 -eq $? ]]; then
115 kill -s SIGTERM ${srs_pid} 2>/dev/null 115 kill -s SIGTERM ${srs_pid} 2>/dev/null
116 ret=$?; if [[ 0 -ne $ret ]]; then failed_msg "send signal SIGTERM failed ret=$ret"; return $ret; fi 116 ret=$?; if [[ 0 -ne $ret ]]; then failed_msg "send signal SIGTERM failed ret=$ret"; return $ret; fi
117 - sleep 0.1 117 + sleep 0.3
118 else 118 else
119 ok_msg "SRS stopped by SIGTERM" 119 ok_msg "SRS stopped by SIGTERM"
120 # delete the pid file when stop success. 120 # delete the pid file when stop success.
@@ -52,6 +52,7 @@ using namespace std; @@ -52,6 +52,7 @@ using namespace std;
52 SrsFFMPEG::SrsFFMPEG(std::string ffmpeg_bin) 52 SrsFFMPEG::SrsFFMPEG(std::string ffmpeg_bin)
53 { 53 {
54 started = false; 54 started = false;
  55 + fast_stopped = false;
55 pid = -1; 56 pid = -1;
56 ffmpeg = ffmpeg_bin; 57 ffmpeg = ffmpeg_bin;
57 58
@@ -484,6 +485,11 @@ int SrsFFMPEG::cycle() @@ -484,6 +485,11 @@ int SrsFFMPEG::cycle()
484 return ret; 485 return ret;
485 } 486 }
486 487
  488 + // ffmpeg is prepare to stop, donot cycle.
  489 + if (fast_stopped) {
  490 + return ret;
  491 + }
  492 +
487 int status = 0; 493 int status = 0;
488 pid_t p = waitpid(pid, &status, WNOHANG); 494 pid_t p = waitpid(pid, &status, WNOHANG);
489 495
@@ -524,6 +530,27 @@ void SrsFFMPEG::stop() @@ -524,6 +530,27 @@ void SrsFFMPEG::stop()
524 started = false; 530 started = false;
525 } 531 }
526 532
  533 +void SrsFFMPEG::fast_stop()
  534 +{
  535 + int ret = ERROR_SUCCESS;
  536 +
  537 + if (!started) {
  538 + return;
  539 + }
  540 +
  541 + if (pid <= 0) {
  542 + return;
  543 + }
  544 +
  545 + if (kill(pid, SIGTERM) < 0) {
  546 + ret = ERROR_SYSTEM_KILL;
  547 + srs_warn("ignore fast stop ffmpeg failed, pid=%d. ret=%d", pid, ret);
  548 + return;
  549 + }
  550 +
  551 + return;
  552 +}
  553 +
527 #endif 554 #endif
528 555
529 556
@@ -45,6 +45,8 @@ class SrsFFMPEG @@ -45,6 +45,8 @@ class SrsFFMPEG
45 { 45 {
46 private: 46 private:
47 bool started; 47 bool started;
  48 + // whether SIGINT send but need to wait or SIGKILL.
  49 + bool fast_stopped;
48 pid_t pid; 50 pid_t pid;
49 private: 51 private:
50 std::string log_file; 52 std::string log_file;
@@ -83,7 +85,25 @@ public: @@ -83,7 +85,25 @@ public:
83 virtual int initialize_copy(); 85 virtual int initialize_copy();
84 virtual int start(); 86 virtual int start();
85 virtual int cycle(); 87 virtual int cycle();
  88 + /**
  89 + * send SIGTERM then SIGKILL to ensure the process stopped.
  90 + * the stop will wait [0, SRS_PROCESS_QUIT_TIMEOUT_MS] depends on the
  91 + * process quit timeout.
  92 + * @remark use fast_stop before stop one by one, when got lots of process to quit.
  93 + */
86 virtual void stop(); 94 virtual void stop();
  95 +public:
  96 + /**
  97 + * the fast stop is to send a SIGTERM.
  98 + * for example, the ingesters owner lots of FFMPEG, it will take a long time
  99 + * to stop one by one, instead the ingesters can fast_stop all FFMPEG, then
  100 + * wait one by one to stop, it's more faster.
  101 + * @remark user must use stop() to ensure the ffmpeg to stopped.
  102 + * @remark we got N processes to stop, compare the time we spend,
  103 + * when use stop without fast_stop, we spend maybe [0, SRS_PROCESS_QUIT_TIMEOUT_MS * N]
  104 + * but use fast_stop then stop, the time is almost [0, SRS_PROCESS_QUIT_TIMEOUT_MS].
  105 + */
  106 + virtual void fast_stop();
87 }; 107 };
88 108
89 #endif 109 #endif
@@ -51,6 +51,11 @@ SrsIngesterFFMPEG::~SrsIngesterFFMPEG() @@ -51,6 +51,11 @@ SrsIngesterFFMPEG::~SrsIngesterFFMPEG()
51 srs_freep(ffmpeg); 51 srs_freep(ffmpeg);
52 } 52 }
53 53
  54 +void SrsIngesterFFMPEG::fast_stop()
  55 +{
  56 + ffmpeg->fast_stop();
  57 +}
  58 +
54 SrsIngester::SrsIngester() 59 SrsIngester::SrsIngester()
55 { 60 {
56 _srs_config->subscribe(this); 61 _srs_config->subscribe(this);
@@ -161,6 +166,18 @@ int SrsIngester::parse_engines(SrsConfDirective* vhost, SrsConfDirective* ingest @@ -161,6 +166,18 @@ int SrsIngester::parse_engines(SrsConfDirective* vhost, SrsConfDirective* ingest
161 166
162 void SrsIngester::dispose() 167 void SrsIngester::dispose()
163 { 168 {
  169 + // first, use fast stop to notice all FFMPEG to quit gracefully.
  170 + std::vector<SrsIngesterFFMPEG*>::iterator it;
  171 + for (it = ingesters.begin(); it != ingesters.end(); ++it) {
  172 + SrsIngesterFFMPEG* ingester = *it;
  173 + ingester->fast_stop();
  174 + }
  175 +
  176 + if (!ingesters.empty()) {
  177 + srs_trace("fast stop all ingesters ok.");
  178 + }
  179 +
  180 + // then, use stop to wait FFMPEG quit one by one and send SIGKILL if needed.
164 stop(); 181 stop();
165 } 182 }
166 183
@@ -52,6 +52,9 @@ public: @@ -52,6 +52,9 @@ public:
52 52
53 SrsIngesterFFMPEG(SrsFFMPEG* _ffmpeg, std::string _vhost, std::string _id); 53 SrsIngesterFFMPEG(SrsFFMPEG* _ffmpeg, std::string _vhost, std::string _id);
54 virtual ~SrsIngesterFFMPEG(); 54 virtual ~SrsIngesterFFMPEG();
  55 +
  56 + // @see SrsFFMPEG.fast_stop().
  57 + virtual void fast_stop();
55 }; 58 };
56 59
57 /** 60 /**
@@ -44,6 +44,9 @@ using namespace std; @@ -44,6 +44,9 @@ using namespace std;
44 #include <srs_app_json.hpp> 44 #include <srs_app_json.hpp>
45 #include <srs_kernel_stream.hpp> 45 #include <srs_kernel_stream.hpp>
46 46
  47 +// the longest time to wait for a process to quit.
  48 +#define SRS_PROCESS_QUIT_TIMEOUT_MS 1000
  49 +
47 int srs_socket_connect(string server, int port, int64_t timeout, st_netfd_t* pstfd) 50 int srs_socket_connect(string server, int port, int64_t timeout, st_netfd_t* pstfd)
48 { 51 {
49 int ret = ERROR_SUCCESS; 52 int ret = ERROR_SUCCESS;
@@ -223,7 +226,6 @@ void srs_parse_endpoint(string ip_port, string& ip, int& port) @@ -223,7 +226,6 @@ void srs_parse_endpoint(string ip_port, string& ip, int& port)
223 port = ::atoi(the_port.c_str()); 226 port = ::atoi(the_port.c_str());
224 } 227 }
225 228
226 -#define SRS_PROCESS_QUIT_TIMEOUT_MS 1000  
227 int srs_kill_forced(int& pid) 229 int srs_kill_forced(int& pid)
228 { 230 {
229 int ret = ERROR_SUCCESS; 231 int ret = ERROR_SUCCESS;
@@ -232,13 +234,13 @@ int srs_kill_forced(int& pid) @@ -232,13 +234,13 @@ int srs_kill_forced(int& pid)
232 return ret; 234 return ret;
233 } 235 }
234 236
235 - // first, try kill by SIGINT.  
236 - if (kill(pid, SIGINT) < 0) { 237 + // first, try kill by SIGTERM.
  238 + if (kill(pid, SIGTERM) < 0) {
237 return ERROR_SYSTEM_KILL; 239 return ERROR_SYSTEM_KILL;
238 } 240 }
239 241
240 // wait to quit. 242 // wait to quit.
241 - srs_trace("send SIGINT to pid=%d", pid); 243 + srs_trace("send SIGTERM to pid=%d", pid);
242 for (int i = 0; i < SRS_PROCESS_QUIT_TIMEOUT_MS / 10; i++) { 244 for (int i = 0; i < SRS_PROCESS_QUIT_TIMEOUT_MS / 10; i++) {
243 int status = 0; 245 int status = 0;
244 pid_t qpid = -1; 246 pid_t qpid = -1;
@@ -253,7 +255,7 @@ int srs_kill_forced(int& pid) @@ -253,7 +255,7 @@ int srs_kill_forced(int& pid)
253 } 255 }
254 256
255 // killed, set pid to -1. 257 // killed, set pid to -1.
256 - srs_trace("SIGINT stop process pid=%d ok.", pid); 258 + srs_trace("SIGTERM stop process pid=%d ok.", pid);
257 pid = -1; 259 pid = -1;
258 260
259 return ret; 261 return ret;