it is near to finish

@@ -6,6 +6,7 @@
 #include <vector>
 #include <string.h>
 #include <list>
+#include <deque>
 
 
 bool only_print = false;
@@ -19,7 +20,7 @@ enum media_type{
 
 enum timestamp_type{
 	tt_start = 0,
-	tt_end = 1;
+	tt_end = 1,
 };
 
 class fileinfo {
@@ -39,8 +40,6 @@ public:
 	string name;
 
 	float duration;
-	float cur_start;
-	float cur_end;
 	int index;
 	media_type m_type;
 	timestamp_type t_type;
@@ -163,10 +162,10 @@ void split_audio(const char * audiofile, float audio_start, float duration, char
 	run_shell_cmd(buf);
 }
 
-void get_video_first_frame_jpeg(fileinfo video, char * destfile)
+void get_video_first_frame_jpeg(const char * video, const char * destfile)
 {
 	char buf[2048];
-	sprintf(buf, "ffmpeg -y -i %s -vframes 1 -ss 0 -f mjpeg -an %s", video.name.c_str(), destfile);
+	sprintf(buf, "ffmpeg -y -i %s -vframes 1 -ss 0 -f mjpeg -an %s", video, destfile);
 	run_shell_cmd(buf);
 }
 
@@ -425,17 +424,21 @@ public:
 	int process_files(const char * output_dest_file);
 
 protected:
+	void add_media_info(media_info m);
 	void add_media_infos();
 	void init();
-	void adjust_va_timestamp();
-	void merge_left_audio();
-	int process_video_ahead_of_audio();
-	int process_video_behind_audio();
-	int process_video_align_audio();
-	int process_video_loop();
+	int merge_audio_video(vector<media_info> & files);
+	int merge_audio_pic(vector<media_info> & files);
+	int find_video_between_the_audio();
+	int find_video_end();
+	bool is_audio_start(int index);
+	int split_audio(int index_from, vector<media_info> & result);
+	void get_front_info(int index_to, vector<media_info> &cur_processing);
 	int process_va();
 
 protected:
+	list <media_info> sorted_media;
+	deque<media_info> sorted_infos;
 	vector<media_info>  media_infos;
 	vector<string > merged_files;
 	vector<string>  tmp_files;
@@ -448,20 +451,23 @@ protected:
 	int audio_start;
 	fileinfo audio;
 	fileinfo video;
-private:
-
-	void add_media_info(media_info m)
-	{
-		throw std::logic_error("The method or operation is not implemented.");
-	}
-
-
 };
 
 MergeProcess::MergeProcess(){
 	init();
 }
 
+void MergeProcess::add_media_info(media_info m)
+{
+	list<media_info>::iterator it = sorted_media.begin();
+	for (; it != sorted_media.end(); it++){
+		if (it->type_time > m.type_time){
+			break;
+		}
+	}
+	sorted_media.insert(it, m);
+}
+
 void MergeProcess::add_media_infos()
 {
 	for (int i = 0; i < media_files.size(); i++) {
@@ -474,14 +480,17 @@ void MergeProcess::add_media_infos()
 		m.m_type = f.m_type;
 		m.t_type = tt_start;
 		m.duration = f.end_time - f.start_time;
-		m.cur_start = 0;
-		m.cur_end = m.duration;
 		m.type_time = m.start_time;
 		add_media_info(m);
 		m.t_type = tt_end;
 		m.type_time = m.end_time;
 		add_media_info(m);
 	}
+
+	list<media_info>::iterator it = sorted_media.begin();
+	for (; it != sorted_media.end(); it++){
+		sorted_infos.push_back(*it);
+	}
 }
 
 void MergeProcess::init()
@@ -501,147 +510,46 @@ void MergeProcess::init()
 	audio_index = 0;
 }
 
-void MergeProcess::adjust_va_timestamp()
-{
-	fileinfo audio = filesaudio[audio_index];
-	float audio_start = audio.start_time;
 
-	for (int j = audio_index; j < filesaudio.size(); j++){
-		filesaudio[j].start_time -= audio_start;
-		filesaudio[j].end_time -= audio_start;
-	}
-
-	for (int j = nv; j < filesvideo.size(); j++) {
-		filesvideo[j].start_time -= audio_start;
-		filesvideo[j].end_time -= audio_start;
-	}
-}
-
-
-void MergeProcess::merge_left_audio()
+int MergeProcess::merge_audio_video(vector<media_info> & files)
 {
-	if (audio_start < audio.end_time){
-		sprintf(destfile, "%d_%s", nf, audio.name.c_str());
-		split_audio(audio.name.c_str(), audio_start, audio.end_time - audio_start, destfile);
-		tmp_files.push_back(destfile);
-
-		sprintf(destfile, "%d.ts", nf);
-		if (nv < filesvideo.size()) {
-			fileinfo video = filesvideo[nv];
-			sprintf(pic_file, "%s.jpg", video.name.c_str());
-			get_video_first_frame_jpeg(video, pic_file);
-			tmp_files.push_back(pic_file);
-		}
-		else {
-			strcpy(pic_file, blank_pic_file);
-		}
-		merge_audio_pic(audio, nf, pic_file, destfile);
-		merged_files.push_back(destfile);
-		nf++;
-	}
-}
-
-int  MergeProcess::process_video_ahead_of_audio()
-{
-	if (video.end_time < audio_start + 0.1) {
-		sprintf(destfile, "%d_silence.aac", nf);
-		split_audio(silence_aac_file, 0, video.end_time - video.start_time, destfile);
-		tmp_files.push_back(destfile);
-		sprintf(destfile, "%d.ts", nf);
-		merge_audio_video(destfile, video.name.c_str(), destfile);
-		merged_files.push_back(destfile);
-		nf++;
-		return 0; //for next video
-	}
-	else {
-		// combine a audio file for the video
-		double silence_audio_start = audio.end_time;//maybe need append silence 
-		double silence_audio_end = video.end_time;
+	vector<std::string > merge_audio_files;
+	int nsilence = 0;
 
-		bool need_append_silence = true;
-		bool to_next_video = false;
+	media_info video = files[0];
+	float start_time = video.start_time;
 
-		vector<std::string > merge_audio_files;
-		sprintf(destfile, "%d_0_silence.aac", nf);//a duration of silence
-		split_audio(silence_aac_file, 0, audio_start - video.start_time, destfile);
-		merge_audio_files.push_back(destfile);
-		tmp_files.push_back(destfile);
-
-		if (audio.end_time < video.end_time + 0.1 && audio.end_time > video.end_time - 0.1) {
-			merge_audio_files.push_back(audio.name); //whole audio file,just fit
-			audio_start = audio.end_time + 0.1;
-			need_append_silence = false;
-		}
-		else if (audio.end_time > video.end_time){ //split part of audio file
-			sprintf(destfile, "%d_%s", nf, audio.name.c_str());
-			merge_audio_files.push_back(destfile);
-			tmp_files.push_back(destfile);
-			split_audio(audio.name.c_str(), audio_start, video.end_time - audio_start, destfile);
-			audio_start = video.end_time;
-			need_append_silence = false;
+	for (int i = 1; i < files.size() - 1; i += 2){
+		media_info audio = files[i];
+		media_info audio_end = files[i + 1];
+		if (audio.type_time - start_time > 0.1){
+			sprintf(audio_file, "%d_%d_silence.aac", nf, nsilence++);//a duration of silence
+			::split_audio(silence_aac_file, 0, audio.type_time - start_time, audio_file);
+			merge_audio_files.push_back(silence_aac_file);
+			tmp_files.push_back(silence_aac_file);
 		}
-		else {
-			merge_audio_files.push_back(audio.name);
-
-			for (; audio_index + 1 < filesaudio.size(); audio_index++){//since video is not finished,try find next audio
-				audio = filesaudio[audio_index + 1];
-				if (audio.start_time < video.end_time) {//next audio should split to fit the video
-					silence_audio_end = audio.start_time;
-
-					sprintf(destfile, "%d_%d_silence.aac", nf, audio_index);
-					split_audio(silence_aac_file, 0, silence_audio_end - silence_audio_start, destfile);
-					merge_audio_files.push_back(destfile);
-					tmp_files.push_back(destfile);
-
-					if (audio.end_time > video.end_time - 0.1 && audio.end_time < video.end_time + 0.1) {//just match
-						merge_audio_files.push_back(audio.name);
-						need_append_silence = false;
-						audio_start = audio.end_time + 0.1;
-						audio_index++;//this audio is used
-						break;
-					}
-					if (audio.end_time > video.end_time){
-						sprintf(destfile, "%d_%s", nf, audio.name.c_str());
-						merge_audio_files.push_back(destfile);
-						tmp_files.push_back(destfile);
-						split_audio(audio.name.c_str(), 0, video.end_time - audio.start_time, destfile);
-						need_append_silence = false;
-						//adjust timecode for the audio is part left
-						float cur_audio_start = video.end_time - audio.start_time;
 
-						audio_start = audio.start_time;
+		if (audio.type_time - audio.start_time > 0.10 || audio_end.end_time - audio.type_time > 0.10) {
+			sprintf(audio_file, "%d_%s", nf, audio.name.c_str());
+			::split_audio(audio.name.c_str(), audio.type_time - audio.start_time, audio_end.type_time - audio.start_time, audio_file);
+			tmp_files.push_back(audio_file);
 
-						for (int j = audio_index + 1; j < filesaudio.size(); j++){
-							filesaudio[j].start_time -= audio_start;
-							filesaudio[j].end_time -= audio_start;
 		}
-
-						for (int j = nv; j < filesvideo.size(); j++) {
-							filesvideo[j].start_time -= audio_start;
-							filesvideo[j].end_time -= audio_start;
+		else{
+			strcpy(audio_file, audio.name.c_str());
 		}
-						audio_index++;
-						audio = filesaudio[audio_index];
-						audio_start = cur_audio_start;
 
-						to_next_video = true;
+		start_time = audio_end.type_time;
+		merge_audio_files.push_back(audio_file);
 
-						break;
+		if (i == files.size() - 2){
+			if (video.end_time - audio_end.type_time > 0.1){
+				sprintf(audio_file, "%d_%d_silence.aac", nf, nsilence++);//a duration of silence
+				::split_audio(silence_aac_file, 0, video.end_time - audio_end.type_time, audio_file);
+				merge_audio_files.push_back(silence_aac_file);
+				tmp_files.push_back(silence_aac_file);
 			}
-					merge_audio_files.push_back(audio.name);//whole audio should be appended
-					silence_audio_start = audio.end_time; //adjust the silence start
 		}
-				else {
-					break;//no need for next audio
-				}
-			}//end audio find for the video
-		}//end else
-
-		if (need_append_silence) {
-			sprintf(destfile, "%d_silence.aac", nf);
-			split_audio(silence_aac_file, 0, silence_audio_end - silence_audio_start, destfile);
-			merge_audio_files.push_back(destfile);
-			tmp_files.push_back(destfile);
 	}
 
 	sprintf(audio_file, "%d_merged.aac", nf);
@@ -649,198 +557,159 @@ int  MergeProcess::process_video_ahead_of_audio()
 
 
 	sprintf(destfile, "%d.ts", nf);
-		merge_audio_video(audio_file, video.name.c_str(), destfile);
+	::merge_audio_video(audio_file, video.name.c_str(), destfile);
 	merged_files.push_back(destfile);
 	nf++;
 
-		if (!to_next_video){
-			nv++;//this video is processed
-			return 1;
-		}
-	}//end need combine
 	return 0;
 }
 
-int MergeProcess::process_video_behind_audio()
+int MergeProcess::merge_audio_pic(vector<media_info> & files)
 {
+	media_info audio = files[0];
+	media_info audio_end = files[1];
+	
+	if (audio.type_time - audio.start_time > 0.10 || audio_end.end_time - audio.type_time > 0.10) {
 		sprintf(audio_file, "%d_%s", nf, audio.name.c_str());
-	if (video.start_time < audio.end_time - 0.1){
-		split_audio(audio.name.c_str(), audio_start, video.start_time - audio_start, audio_file);
-		audio_start = video.start_time;
+		::split_audio(audio.name.c_str(), audio.type_time - audio.start_time, audio_end.type_time - audio.start_time, audio_file);
+		tmp_files.push_back(audio_file);
 	}
-	else {
-		split_audio(audio.name.c_str(), audio_start, audio.end_time - audio_start, audio_file);
+	else{
+		strcpy(audio_file, audio.name.c_str());
 	}
-	tmp_files.push_back(audio_file);
 	
-	sprintf(pic_file, "%s.jpg", video.name.c_str());
-	get_video_first_frame_jpeg(video, pic_file);
+	sprintf(destfile, "%d.ts", nf);
+
+	int i = 0;
+	for (; i < sorted_infos.size(); i++){
+		if (sorted_infos[i].m_type == mt_video){
+			string name = sorted_infos[i].name;
+			sprintf(pic_file, "%s.jpg", name.c_str());
+			get_video_first_frame_jpeg(name.c_str(), pic_file);
 			tmp_files.push_back(pic_file);
+			break;
+		}
+	}
+	if (i == sorted_infos.size()){
+		strcpy(pic_file, blank_pic_file);
+	}
 
-	sprintf(destfile, "%d.ts", nf);
-	merge_audio_pic(audio_file, pic_file, destfile);
+	::merge_audio_pic(audio_file, pic_file, destfile);
 	merged_files.push_back(destfile);
 	nf++;
 
-	if (video.start_time >= audio.end_time - 0.1){//all audio file no video, to next audio
-		audio_start = audio.end_time + 0.1;//no audio left
-		return 1;
-	}
 	return 0;
 }
 
-int MergeProcess::process_video_align_audio()
+int MergeProcess::find_video_between_the_audio()
 {
-	if (audio.end_time > video.end_time){ //this video finish, to next video
-		sprintf(destfile, "%d_%s", nf, audio.name.c_str());
-		split_audio(audio.name.c_str(), video.start_time, video.end_time - video.start_time, destfile);
-		tmp_files.push_back(destfile);
-
-		audio_start = video.end_time;
-		sprintf(destfile, "%d.ts", nf);
-		merge_audio_video(audio, nf, video, destfile);
-		merged_files.push_back(destfile);
-		nf++;
+	int index = sorted_infos[0].index;
+	int video_index = 0;
+	for (int i = 1; i < sorted_infos.size(); i++){
+		if (sorted_infos[i].index == index)
+			break;
+		if (sorted_infos[i].m_type == mt_video)
+		{
+			video_index = i;
+			break;
 		}
-	else if (video.end_time - audio.end_time < 0.1){//just fine, this audio file finish
-		sprintf(destfile, "%d_%s", nf, audio.name.c_str());
-		split_audio(audio.name.c_str(), video.start_time, audio.end_time - video.start_time, destfile);
-		tmp_files.push_back(destfile);
-
-		sprintf(destfile, "%d.ts", nf);
-		merge_audio_video(audio, nf, video, destfile);
-		merged_files.push_back(destfile);
-		audio_start = audio.end_time + 0.1;//no audio left
-		nf++;
-		nv++;//this video is used
-		return 1;
 	}
-	else { // this audio finish,add silence and/or next audio
-		sprintf(destfile, "%d_%s", nf, audio.name.c_str());
-		split_audio(audio.name.c_str(), video.start_time, audio.end_time - video.start_time, destfile);
-		vector<std::string > merge_audio_files;
-		merge_audio_files.push_back(destfile);
-		tmp_files.push_back(destfile);
-
-		audio_start = audio.end_time + 0.1;
-
-		double silence_audio_start = audio.end_time;
-		double silence_audio_end = video.end_time;
-
-		bool need_silence = true;
-		bool to_next_video = false;
-		for (; audio_index + 1 < filesaudio.size(); audio_index++){//since video is not finished,try find next audio
-			audio = filesaudio[audio_index + 1];
-			if (audio.start_time < video.end_time) {//next audio should split to fit the video
-				silence_audio_end = audio.start_time;
-
-				sprintf(destfile, "%d_%d_silence.aac", nf, audio_index);
-				split_audio(silence_aac_file, 0, silence_audio_end - silence_audio_start, destfile);
-				merge_audio_files.push_back(destfile);
-				tmp_files.push_back(destfile);
-
-				if (audio.end_time > video.end_time - 0.1 && audio.end_time < video.end_time + 0.1) {//just match
-					merge_audio_files.push_back(audio.name);
-					need_silence = false;
-					audio_start = audio.end_time + 0.1;
-					audio_index++;
+	return video_index;
+}
+
+int MergeProcess::find_video_end()
+{
+	int index = sorted_infos[0].index;
+	int video_index = 0;
+	for (int i = 1; i < sorted_infos.size(); i++){
+		if (sorted_infos[i].index == index){
+			video_index = i;
 			break;
 		}
-				if (audio.end_time > video.end_time){
-					sprintf(destfile, "%d_%s", nf, audio.name.c_str());
-					merge_audio_files.push_back(destfile);
-					tmp_files.push_back(destfile);
-					split_audio(audio.name.c_str(), 0, video.end_time - audio.start_time, destfile);
-					need_silence = false;
-					//adjust timecode for the audio is part left
-					float cur_audio_start = video.end_time - audio.start_time;
+	}
+	return video_index;
+}
 
-					audio_start = audio.start_time;
+bool MergeProcess::is_audio_start(int index)
+{
+	return (sorted_infos[index].m_type == mt_audio && sorted_infos[index].t_type == tt_start);
+}
 
-					for (int j = audio_index + 1; j < filesaudio.size(); j++){
-						filesaudio[j].start_time -= audio_start;
-						filesaudio[j].end_time -= audio_start;
-					}
 
-					for (int j = nv; j < filesvideo.size(); j++) {
-						filesvideo[j].start_time -= audio_start;
-						filesvideo[j].end_time -= audio_start;
-					}
-					audio_index++;
-					audio = filesaudio[audio_index];
-					audio_start = cur_audio_start;
+int MergeProcess::split_audio(int index_from, vector<media_info> & result)
+{
+	media_info audio = sorted_infos[index_from];
+	media_info video = sorted_infos[index_from + 1];
 
-					to_next_video = true;
+	result.clear();
 
-					break;
+	for (int i = 0; i < index_from; i++){
+		result.push_back(sorted_infos[i]);
 	}
-				merge_audio_files.push_back(audio.name);//whole audio should be appended
-				silence_audio_start = audio.end_time; //adjust the silence start
-				audio_start = audio.end_time + 0.1;
+
+	if (sorted_infos[index_from + 2].index == sorted_infos[index_from].index){
+		if (sorted_infos[index_from].end_time - sorted_infos[index_from + 1].start_time < 0.1){//no need to split
+			result.push_back(sorted_infos[index_from + 2]);//put the audio end to the result
+			for (int i = 0; i < index_from + 2; i++){
+				sorted_infos.pop_front();
 			}
-			else {
-				break;//no need for next audio
+			sorted_infos.push_front(video); 
+			return 0;
 		}
 	}
 
-		if (need_silence) {
-			sprintf(destfile, "%d_silence.aac", nf);
-			split_audio(silence_aac_file, 0, silence_audio_end - silence_audio_start, destfile);
-			merge_audio_files.push_back(destfile);
-			tmp_files.push_back(destfile);
-		}
+	audio.t_type = tt_end;
+	audio.type_time = video.start_time;
+	result.push_back(audio);
 
-		sprintf(audio_file, "%d_merged.aac", nf);
-		merge_audio_file(merge_audio_files, audio_file);
-
-
-		sprintf(destfile, "%d.ts", nf);
-		merge_audio_video(audio_file, video.name.c_str(), destfile);
-		merged_files.push_back(destfile);
-		nf++;
-
-		if (!to_next_video){
-			nv++;
-			return 1;
-		}
+	for (int i = 0; i < index_from + 1; i++){
+		sorted_infos.pop_front();
 	}
+
+	audio.t_type = tt_start;
+	sorted_infos.push_front(audio);
+	sorted_infos.push_front(video);
 	return 0;
 }
 
-int MergeProcess::process_video_loop()
+void MergeProcess::get_front_info(int index_to, vector<media_info> &cur_processing)
 {
-	for (; nv < filesvideo.size(); nv++) {
-		video = filesvideo[nv];
-
-		if (video.start_time < audio_start - 0.1)
-		{//video is much more ahead of audio,try padding silence first
-			if (process_video_ahead_of_audio())
-				break;
-		}//end video is ahead of audio
-		if (video.start_time - audio_start > 0.1) {//video is behind audio too much
-			if (process_video_behind_audio())
-				break;
-		}
-		else if (audio_start - video.start_time < 0.1){
-			if (process_video_align_audio())
-				break;
+	cur_processing.clear();
+	for (int i = 0; i <= index_to; i++){
+		cur_processing.push_back(sorted_infos[i]);
 	}
+	for (int i = 0; i <= index_to; i++){
+		sorted_infos.pop_front();
 	}
-	return 0;
 }
 
 int MergeProcess::process_va()
 {
-	for (audio_index = 0; audio_index < filesaudio.size(); audio_index++){  //
-
-		adjust_va_timestamp();
-
-		audio = filesaudio[audio_index];
-		audio_start = 0;//for a new processing audio,the start is 0
-
-		process_video_loop();
-
-		merge_left_audio();
+	vector<media_info> cur_processing;
+	while (sorted_infos.size())
+	{
+		media_type mt = sorted_infos[0].m_type;
+		if (mt == mt_audio){
+			int index = find_video_between_the_audio();
+			if (index > 0) //have_video
+			{
+				split_audio(index, cur_processing);
+			}
+			else {
+				get_front_info(1, cur_processing);
+			}
+			merge_audio_pic(cur_processing);
+		}
+		else{
+			int index = find_video_end();
+    		if (is_audio_start(index - 1)) {
+				split_audio(index - 1, cur_processing);
+			}
+			else {
+				get_front_info(index, cur_processing);
+			}
+			merge_audio_video(cur_processing);
+		}
 	}
 	return 0;
 }
@@ -850,43 +719,55 @@ int MergeProcess::process_files(const char * output_dest_file)
 {
 	//don't split video, for a video, using merged audios to mix with it
 	//for audio, mix with video or jpg
+	if (!media_files.size()){
+		return 0;
+	}
 
-	if (filesvideo.size()) {//has video files
-		if (filesaudio.size()){
-			process_va(); //process the case both audio and video files exist
+	// judge if it is only one type
+	media_type mt = media_files[0].m_type;
+	bool only_one_type = true;
+	for (int i = 1; i < media_files.size(); i++){
+		if (mt != media_files[i].m_type){
+			only_one_type = false;
+			break;
 		}
-		else{//only video
-			if (filesvideo.size() == 1){
-				fileinfo video = filesvideo[0];
-				merge_video_silence(video, silence_aac_file, "dest.ts");
+	}
+
+	if (only_one_type){
+		if (mt == mt_audio) {
+			if (media_files.size() == 1){
+				fileinfo audio = media_files[0];
+				::merge_audio_pic(audio, blank_pic_file, "dest.ts");
 				return 0;
 			}
 
-			for (int i = 0; i < filesvideo.size(); i++){
-				fileinfo video = filesvideo[i];
+			for (int i = 0; i < media_files.size(); i++){
+				fileinfo audio = media_files[i];
 				sprintf(destfile, "%d.ts", nf);
-				merge_video_silence(video, silence_aac_file, destfile);
+				::merge_audio_pic(audio, blank_pic_file, destfile);
 				merged_files.push_back(destfile);
 				nf++;
 			}
 		}
-	}
-	else {//only audio
-
-		if (filesaudio.size() == 1){
-			fileinfo audio = filesaudio[0];
-			merge_audio_pic(audio, blank_pic_file, "dest.ts");
+		else {
+			if (media_files.size() == 1){
+				fileinfo video = media_files[0];
+				merge_video_silence(video, silence_aac_file, "dest.ts");
 				return 0;
 			}
 
-		for (int i = 0; i < filesaudio.size(); i++){
-			fileinfo audio = filesaudio[i];
+			for (int i = 0; i < media_files.size(); i++){
+				fileinfo video = media_files[i];
 				sprintf(destfile, "%d.ts", nf);
-			merge_audio_pic(audio, blank_pic_file, destfile);
+				merge_video_silence(video, silence_aac_file, destfile);
 				merged_files.push_back(destfile);
 				nf++;
 			}
 		}
+	}
+	else {
+		process_va();
+	}
 
 	concate_files(merged_files, "m.ts");
 	tmp_files.push_back("m.ts");
@@ -986,7 +867,7 @@ int main(int argc, char * argv[])
 
 	load_codec_param();
 
-	MergeProcess mp(media_files[type_audio], media_files[type_video]);
+	MergeProcess mp;
 	mp.process_files("dest.ts");
 
 	return 0;