xuning

录音实现

... ... @@ -3,6 +3,8 @@
android:versionCode="1"
android:versionName="1.1">
<uses-permission android:name="android.permission.CAMERA" />
<uses-permission android:name="android.permission.RECORD_AUDIO" />
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
<uses-feature android:name="android.hardware.camera2.full" />
<uses-feature android:name="android.hardware.camera" android:required="false" />
... ...
... ... @@ -19,6 +19,7 @@ import android.app.Activity;
import android.content.pm.PackageManager;
import android.graphics.PixelFormat;
import android.os.Bundle;
import android.os.Environment;
import android.util.Log;
import android.view.Surface;
import android.view.SurfaceHolder;
... ... @@ -28,6 +29,7 @@ import android.view.WindowManager;
import android.widget.AdapterView;
import android.widget.Button;
import android.widget.Spinner;
import android.widget.Toast;
import android.support.v4.app.ActivityCompat;
import android.support.v4.content.ContextCompat;
... ... @@ -35,9 +37,11 @@ import android.support.v4.content.ContextCompat;
public class MainActivity extends Activity implements SurfaceHolder.Callback
{
public static final int REQUEST_CAMERA = 100;
public static final int REQUEST_RECORD_AUDIO = 101;
private YOLO11Ncnn yolo11ncnn = new YOLO11Ncnn();
private int facing = 0;
private boolean isRecording = false;
private Spinner spinnerTask;
private Spinner spinnerModel;
... ... @@ -77,6 +81,36 @@ public class MainActivity extends Activity implements SurfaceHolder.Callback
}
});
final Button buttonRecord = (Button) findViewById(R.id.buttonRecord);
buttonRecord.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View arg0) {
if (!isRecording) {
// 检查录音权限
if (ContextCompat.checkSelfPermission(getApplicationContext(), Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_DENIED) {
ActivityCompat.requestPermissions(MainActivity.this, new String[] {Manifest.permission.RECORD_AUDIO}, REQUEST_RECORD_AUDIO);
return;
}
// 生成文件名
String timestamp = String.valueOf(System.currentTimeMillis());
String filepath = Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_MOVIES).getAbsolutePath() + "/yolo11_record_" + timestamp + ".mp4";
if (yolo11ncnn.startRecording(filepath)) {
isRecording = true;
buttonRecord.setText("停止录制");
Toast.makeText(MainActivity.this, "开始录制", Toast.LENGTH_SHORT).show();
}
} else {
if (yolo11ncnn.stopRecording()) {
isRecording = false;
buttonRecord.setText("开始录制");
Toast.makeText(MainActivity.this, "录制已停止", Toast.LENGTH_SHORT).show();
}
}
}
});
spinnerTask = (Spinner) findViewById(R.id.spinnerTask);
spinnerTask.setOnItemSelectedListener(new AdapterView.OnItemSelectedListener() {
@Override
... ...
... ... @@ -23,6 +23,9 @@ public class YOLO11Ncnn
public native boolean openCamera(int facing);
public native boolean closeCamera();
public native boolean setOutputWindow(Surface surface);
public native boolean startRecording(String filepath);
public native boolean stopRecording();
public native boolean isRecording();
static {
System.loadLibrary("yolo11ncnn");
... ...
... ... @@ -10,4 +10,4 @@ find_package(ncnn REQUIRED)
add_library(yolo11ncnn SHARED yolo11ncnn.cpp yolo11.cpp yolo11_det.cpp yolo11_seg.cpp yolo11_pose.cpp yolo11_cls.cpp yolo11_obb.cpp ndkcamera.cpp)
target_link_libraries(yolo11ncnn ncnn ${OpenCV_LIBS} camera2ndk mediandk)
target_link_libraries(yolo11ncnn ncnn ${OpenCV_LIBS} camera2ndk mediandk)
\ No newline at end of file
... ...
... ... @@ -15,10 +15,20 @@
#include "ndkcamera.h"
#include <string>
#include <thread>
#include <queue>
#include <mutex>
#include <android/log.h>
#include <media/NdkMediaCodec.h>
#include <media/NdkMediaMuxer.h>
#include <media/NdkMediaFormat.h>
#include <chrono>
#include <unistd.h>
#include <fcntl.h>
#include <opencv2/core/core.hpp>
#include <sys/time.h>
#include "mat.h"
... ... @@ -434,10 +444,25 @@ NdkCameraWindow::NdkCameraWindow() : NdkCamera()
sensor_manager = ASensorManager_getInstance();
accelerometer_sensor = ASensorManager_getDefaultSensor(sensor_manager, ASENSOR_TYPE_ACCELEROMETER);
// recording
recording_active = false;
recording_thread = nullptr;
video_encoder = nullptr;
audio_encoder = nullptr;
media_muxer = nullptr;
video_track_index = -1;
audio_track_index = -1;
muxer_started = false;
}
NdkCameraWindow::~NdkCameraWindow()
{
// stop recording if active
if (recording_active) {
stopRecording();
}
if (accelerometer_sensor)
{
ASensorEventQueue_disableSensor(sensor_event_queue, accelerometer_sensor);
... ... @@ -769,3 +794,422 @@ void NdkCameraWindow::on_image(const unsigned char* nv21, int nv21_width, int nv
ANativeWindow_unlockAndPost(win);
}
// Recording functions implementation
bool NdkCameraWindow::startRecording(const char* filepath) {
std::lock_guard<std::mutex> lock(recording_mutex);
if (recording_active) {
__android_log_print(ANDROID_LOG_ERROR, "NdkCamera", "Recording already active");
return false;
}
// 保存文件路径
recording_filepath = filepath;
// 创建MediaMuxer - 使用文件描述符
int fd = ::open(recording_filepath.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644);
if (fd < 0) {
__android_log_print(ANDROID_LOG_ERROR, "NdkCamera", "Failed to open file: %s", recording_filepath.c_str());
return false;
}
media_muxer = AMediaMuxer_new(fd, AMEDIAMUXER_OUTPUT_FORMAT_MPEG_4);
if (!media_muxer) {
__android_log_print(ANDROID_LOG_ERROR, "NdkCamera", "Failed to create media muxer");
::close(fd);
return false;
}
// 初始化编码器
if (!setup_video_encoder(640, 480, 30)) {
__android_log_print(ANDROID_LOG_ERROR, "NdkCamera", "Failed to setup video encoder");
AMediaMuxer_delete(media_muxer);
media_muxer = nullptr;
::close(fd);
return false;
}
if (!setup_audio_encoder()) {
__android_log_print(ANDROID_LOG_ERROR, "NdkCamera", "Failed to setup audio encoder");
AMediaCodec_delete(video_encoder);
video_encoder = nullptr;
AMediaMuxer_delete(media_muxer);
media_muxer = nullptr;
::close(fd);
return false;
}
recording_active = true;
video_track_index = -1;
audio_track_index = -1;
muxer_started = false;
// 清空帧队列
while (!video_frame_queue.empty()) {
video_frame_queue.pop();
}
// 创建录制线程
recording_thread = new std::thread(&NdkCameraWindow::recording_worker, this);
__android_log_print(ANDROID_LOG_INFO, "NdkCamera", "Recording started: %s", filepath);
return true;
}
bool NdkCameraWindow::stopRecording() {
{
std::lock_guard<std::mutex> lock(recording_mutex);
if (!recording_active) {
__android_log_print(ANDROID_LOG_WARN, "NdkCamera", "Recording not active");
return false;
}
recording_active = false;
__android_log_print(ANDROID_LOG_INFO, "NdkCamera", "Recording flag set to false");
}
// 等待录制线程结束
if (recording_thread && recording_thread->joinable()) {
__android_log_print(ANDROID_LOG_INFO, "NdkCamera", "Waiting for recording thread to finish...");
recording_thread->join();
delete recording_thread;
recording_thread = nullptr;
__android_log_print(ANDROID_LOG_INFO, "NdkCamera", "Recording thread finished");
}
// 清理资源
if (media_muxer) {
__android_log_print(ANDROID_LOG_INFO, "NdkCamera", "Stopping media muxer");
if (muxer_started) {
AMediaMuxer_stop(media_muxer);
}
AMediaMuxer_delete(media_muxer);
media_muxer = nullptr;
}
if (video_encoder) {
__android_log_print(ANDROID_LOG_INFO, "NdkCamera", "Stopping video encoder");
AMediaCodec_stop(video_encoder);
AMediaCodec_delete(video_encoder);
video_encoder = nullptr;
}
if (audio_encoder) {
__android_log_print(ANDROID_LOG_INFO, "NdkCamera", "Stopping audio encoder");
AMediaCodec_stop(audio_encoder);
AMediaCodec_delete(audio_encoder);
audio_encoder = nullptr;
}
// 清空帧队列
{
std::lock_guard<std::mutex> lock(recording_mutex);
while (!video_frame_queue.empty()) {
video_frame_queue.pop();
}
}
__android_log_print(ANDROID_LOG_INFO, "NdkCamera", "Recording stopped successfully");
return true;
}
bool NdkCameraWindow::isRecording() const {
std::lock_guard<std::mutex> lock(const_cast<std::mutex&>(recording_mutex));
return recording_active;
}
void NdkCameraWindow::recording_worker() {
__android_log_print(ANDROID_LOG_INFO, "NdkCamera", "Recording worker started");
int64_t start_time = get_current_timestamp();
int frame_count = 0;
// 录制工作线程
while (true) {
{
std::lock_guard<std::mutex> lock(recording_mutex);
if (!recording_active) {
break;
}
}
// 处理视频帧队列
std::vector<uint8_t> frame;
bool has_frame = false;
{
std::lock_guard<std::mutex> lock(recording_mutex);
if (!video_frame_queue.empty()) {
frame = video_frame_queue.front();
video_frame_queue.pop();
has_frame = true;
frame_count++;
}
}
if (has_frame && video_encoder) {
// 编码视频帧
encode_video_frame_data(frame.data(), frame.size());
}
// 处理编码器输出
process_encoder_output();
// 记录录制时长
int64_t current_time = get_current_timestamp();
int64_t elapsed_seconds = (current_time - start_time) / 1000000;
// 每5秒记录一次状态
if (elapsed_seconds > 0 && elapsed_seconds % 5 == 0) {
__android_log_print(ANDROID_LOG_INFO, "NdkCamera",
"Recording: %ld seconds, %d frames processed",
elapsed_seconds, frame_count);
}
// 短暂休眠避免过度占用CPU
usleep(10000); // 10ms
}
__android_log_print(ANDROID_LOG_INFO, "NdkCamera", "Recording worker stopping");
// 录制结束时,处理剩余的编码器输出
if (video_encoder) {
process_encoder_output_final();
}
__android_log_print(ANDROID_LOG_INFO, "NdkCamera", "Recording worker finished, total frames: %d", frame_count);
}
bool NdkCameraWindow::setup_video_encoder(int width, int height, int fps) {
// 创建视频编码器
const char* mime_type = "video/avc";
video_encoder = AMediaCodec_createEncoderByType(mime_type);
if (!video_encoder) {
__android_log_print(ANDROID_LOG_ERROR, "NdkCamera", "Failed to create video encoder");
return false;
}
// 配置视频格式
AMediaFormat* format = AMediaFormat_new();
AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, mime_type);
AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, width);
AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, height);
AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_FRAME_RATE, fps);
AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_I_FRAME_INTERVAL, 1);
AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_BIT_RATE, 2000000);
AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT,
21); // COLOR_FormatYUV420SemiPlanar
// 配置编码器
media_status_t status = AMediaCodec_configure(video_encoder, format,
nullptr, nullptr,
AMEDIACODEC_CONFIGURE_FLAG_ENCODE);
AMediaFormat_delete(format);
if (status != AMEDIA_OK) {
__android_log_print(ANDROID_LOG_ERROR, "NdkCamera", "Failed to configure video encoder: %d", status);
AMediaCodec_delete(video_encoder);
video_encoder = nullptr;
return false;
}
// 启动编码器
status = AMediaCodec_start(video_encoder);
if (status != AMEDIA_OK) {
__android_log_print(ANDROID_LOG_ERROR, "NdkCamera", "Failed to start video encoder: %d", status);
AMediaCodec_delete(video_encoder);
video_encoder = nullptr;
return false;
}
__android_log_print(ANDROID_LOG_INFO, "NdkCamera", "Video encoder setup: %dx%d %dfps", width, height, fps);
return true;
}
bool NdkCameraWindow::setup_audio_encoder() {
// 创建音频编码器
const char* mime_type = "audio/mp4a-latm";
audio_encoder = AMediaCodec_createEncoderByType(mime_type);
if (!audio_encoder) {
__android_log_print(ANDROID_LOG_ERROR, "NdkCamera", "Failed to create audio encoder");
return false;
}
// 配置音频格式
AMediaFormat* format = AMediaFormat_new();
AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, mime_type);
AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_SAMPLE_RATE, 44100);
AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_CHANNEL_COUNT, 1);
AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_BIT_RATE, 128000);
AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_AAC_PROFILE, 2); // AAC LC
// 配置编码器
media_status_t status = AMediaCodec_configure(audio_encoder, format,
nullptr, nullptr,
AMEDIACODEC_CONFIGURE_FLAG_ENCODE);
AMediaFormat_delete(format);
if (status != AMEDIA_OK) {
__android_log_print(ANDROID_LOG_ERROR, "NdkCamera", "Failed to configure audio encoder: %d", status);
AMediaCodec_delete(audio_encoder);
audio_encoder = nullptr;
return false;
}
// 启动编码器
status = AMediaCodec_start(audio_encoder);
if (status != AMEDIA_OK) {
__android_log_print(ANDROID_LOG_ERROR, "NdkCamera", "Failed to start audio encoder: %d", status);
AMediaCodec_delete(audio_encoder);
audio_encoder = nullptr;
return false;
}
__android_log_print(ANDROID_LOG_INFO, "NdkCamera", "Audio encoder setup");
return true;
}
void NdkCameraWindow::encode_video_frame(const unsigned char* nv21_data, int width, int height, int64_t timestamp_us) {
if (!video_encoder) {
return;
}
// 检查录制状态
{
std::lock_guard<std::mutex> lock(recording_mutex);
if (!recording_active) {
return;
}
}
// 限制队列大小,避免内存溢出
const size_t max_queue_size = 30; // 最多缓存30帧
std::lock_guard<std::mutex> lock(recording_mutex);
if (video_frame_queue.size() >= max_queue_size) {
// 队列已满,丢弃最旧的一帧
video_frame_queue.pop();
__android_log_print(ANDROID_LOG_WARN, "NdkCamera", "Video frame queue full, dropping frame");
}
// 将视频帧添加到队列供录制线程处理
std::vector<uint8_t> frame_data(nv21_data, nv21_data + width * height * 3 / 2);
video_frame_queue.push(std::move(frame_data));
}
void NdkCameraWindow::encode_video_frame_data(const uint8_t* data, size_t size) {
if (!video_encoder || !recording_active) {
return;
}
// 获取输入缓冲区
ssize_t input_index = AMediaCodec_dequeueInputBuffer(video_encoder, 10000);
if (input_index >= 0) {
size_t buffer_size;
uint8_t* buffer = AMediaCodec_getInputBuffer(video_encoder, input_index, &buffer_size);
if (buffer && buffer_size >= size) {
// 复制数据到缓冲区
memcpy(buffer, data, size);
// 提交缓冲区给编码器
media_status_t status = AMediaCodec_queueInputBuffer(video_encoder, input_index,
0, size,
get_current_timestamp(), 0);
if (status != AMEDIA_OK) {
__android_log_print(ANDROID_LOG_ERROR, "NdkCamera", "Failed to queue input buffer: %d", status);
} else {
__android_log_print(ANDROID_LOG_DEBUG, "NdkCamera", "Queued input buffer: index=%zd, size=%zu",
input_index, size);
}
} else {
__android_log_print(ANDROID_LOG_WARN, "NdkCamera", "Input buffer too small: need=%zu, have=%zu",
size, buffer_size);
}
} else if (input_index == AMEDIACODEC_INFO_TRY_AGAIN_LATER) {
__android_log_print(ANDROID_LOG_DEBUG, "NdkCamera", "No input buffer available, try again later");
} else {
__android_log_print(ANDROID_LOG_ERROR, "NdkCamera", "Failed to get input buffer: %zd", input_index);
}
}
void NdkCameraWindow::process_encoder_output() {
if (!video_encoder || !recording_active) {
return;
}
// 处理输出缓冲区
AMediaCodecBufferInfo info;
ssize_t output_index = AMediaCodec_dequeueOutputBuffer(video_encoder, &info, 10000); // 10ms超时
while (output_index >= 0) {
if (info.flags & AMEDIACODEC_BUFFER_FLAG_CODEC_CONFIG) {
// 配置数据
if (!muxer_started && media_muxer) {
// 添加视频轨道
AMediaFormat* format = AMediaCodec_getOutputFormat(video_encoder);
if (format) {
video_track_index = AMediaMuxer_addTrack(media_muxer, format);
AMediaFormat_delete(format);
__android_log_print(ANDROID_LOG_INFO, "NdkCamera", "Video track added: %d", video_track_index);
// 如果音频轨道也已准备好,启动混合器
if (audio_track_index >= 0 || !audio_encoder) {
media_status_t status = AMediaMuxer_start(media_muxer);
if (status == AMEDIA_OK) {
muxer_started = true;
__android_log_print(ANDROID_LOG_INFO, "NdkCamera", "MediaMuxer started");
} else {
__android_log_print(ANDROID_LOG_ERROR, "NdkCamera", "Failed to start MediaMuxer: %d", status);
}
}
}
}
} else if (info.size > 0) {
// 编码数据
if (muxer_started && media_muxer) {
size_t buffer_size;
uint8_t* buffer = AMediaCodec_getOutputBuffer(video_encoder, output_index, &buffer_size);
if (buffer) {
media_status_t status = AMediaMuxer_writeSampleData(media_muxer, video_track_index, buffer, &info);
if (status != AMEDIA_OK) {
__android_log_print(ANDROID_LOG_ERROR, "NdkCamera", "Failed to write sample data: %d", status);
} else {
__android_log_print(ANDROID_LOG_DEBUG, "NdkCamera", "Wrote sample data: size=%d, pts=%ld",
info.size, info.presentationTimeUs);
}
}
}
}
AMediaCodec_releaseOutputBuffer(video_encoder, output_index, false);
output_index = AMediaCodec_dequeueOutputBuffer(video_encoder, &info, 10000);
}
}
void NdkCameraWindow::process_encoder_output_final() {
if (!video_encoder) {
return;
}
// 处理所有剩余的编码器输出
AMediaCodecBufferInfo info;
ssize_t output_index = AMediaCodec_dequeueOutputBuffer(video_encoder, &info, 1000000); // 1秒超时
while (output_index >= 0) {
if (info.size > 0 && muxer_started && media_muxer) {
size_t buffer_size;
uint8_t* buffer = AMediaCodec_getOutputBuffer(video_encoder, output_index, &buffer_size);
if (buffer) {
AMediaMuxer_writeSampleData(media_muxer, video_track_index, buffer, &info);
}
}
AMediaCodec_releaseOutputBuffer(video_encoder, output_index, false);
output_index = AMediaCodec_dequeueOutputBuffer(video_encoder, &info, 1000000);
}
}
int64_t NdkCameraWindow::get_current_timestamp() {
struct timeval tv;
gettimeofday(&tv, nullptr);
return (int64_t)tv.tv_sec * 1000000 + tv.tv_usec;
}
... ...
... ... @@ -22,6 +22,12 @@
#include <camera/NdkCameraManager.h>
#include <camera/NdkCameraMetadata.h>
#include <media/NdkImageReader.h>
#include <media/NdkMediaCodec.h>
#include <media/NdkMediaMuxer.h>
#include <media/NdkMediaFormat.h>
#include <thread>
#include <mutex>
#include <queue>
#include <opencv2/core/core.hpp>
... ... @@ -67,6 +73,11 @@ public:
virtual void on_image(const unsigned char* nv21, int nv21_width, int nv21_height) const;
// 录制相关方法
bool startRecording(const char* filepath);
bool stopRecording();
bool isRecording() const;
public:
mutable int accelerometer_orientation;
... ... @@ -75,6 +86,30 @@ private:
mutable ASensorEventQueue* sensor_event_queue;
const ASensor* accelerometer_sensor;
ANativeWindow* win;
// 录制相关成员变量
AMediaCodec* video_encoder;
AMediaCodec* audio_encoder;
AMediaMuxer* media_muxer;
std::thread* recording_thread;
std::mutex recording_mutex;
std::queue<std::vector<uint8_t>> video_frame_queue;
bool recording_active;
int video_track_index;
int audio_track_index;
bool muxer_started;
void recording_worker();
bool setup_video_encoder(int width, int height, int fps);
bool setup_audio_encoder();
void encode_video_frame(const unsigned char* nv21_data, int width, int height, int64_t timestamp_us);
void encode_video_frame_data(const uint8_t* data, size_t size);
int64_t get_current_timestamp();
void process_encoder_output();
void process_encoder_output_final();
private:
std::string recording_filepath;
};
#endif // NDKCAMERA_H
... ...
... ... @@ -22,6 +22,9 @@
#include <string>
#include <vector>
#include <thread>
#include <mutex>
#include <queue>
#include <platform.h>
#include <benchmark.h>
... ... @@ -33,6 +36,10 @@
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <media/NdkMediaCodec.h>
#include <media/NdkMediaMuxer.h>
#include <media/NdkMediaFormat.h>
#if __ARM_NEON
#include <arm_neon.h>
#endif // __ARM_NEON
... ... @@ -298,4 +305,39 @@ JNIEXPORT jboolean JNICALL Java_com_tencent_yolo11ncnn_YOLO11Ncnn_setOutputWindo
return JNI_TRUE;
}
// public native boolean startRecording(String filepath);
JNIEXPORT jboolean JNICALL Java_com_tencent_yolo11ncnn_YOLO11Ncnn_startRecording(JNIEnv* env, jobject thiz, jstring filepath)
{
const char* filepath_str = env->GetStringUTFChars(filepath, nullptr);
if (!filepath_str) {
return JNI_FALSE;
}
__android_log_print(ANDROID_LOG_DEBUG, "ncnn", "startRecording %s", filepath_str);
bool result = g_camera->startRecording(filepath_str);
env->ReleaseStringUTFChars(filepath, filepath_str);
return result ? JNI_TRUE : JNI_FALSE;
}
// public native boolean stopRecording();
JNIEXPORT jboolean JNICALL Java_com_tencent_yolo11ncnn_YOLO11Ncnn_stopRecording(JNIEnv* env, jobject thiz)
{
__android_log_print(ANDROID_LOG_DEBUG, "ncnn", "stopRecording");
bool result = g_camera->stopRecording();
return result ? JNI_TRUE : JNI_FALSE;
}
// public native boolean isRecording();
JNIEXPORT jboolean JNICALL Java_com_tencent_yolo11ncnn_YOLO11Ncnn_isRecording(JNIEnv* env, jobject thiz)
{
bool result = g_camera->isRecording();
return result ? JNI_TRUE : JNI_FALSE;
}
}
... ...
... ... @@ -15,6 +15,12 @@
android:layout_height="wrap_content"
android:text="切换摄像头" />
<Button
android:id="@+id/buttonRecord"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="开始录制" />
</LinearLayout>
<LinearLayout
... ...