rvmncnn.cpp 9.2 KB
// Tencent is pleased to support the open source community by making ncnn available.
//
// Copyright (C) 2025 THL A29 Limited, a Tencent company. All rights reserved.
//
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
//
// https://opensource.org/licenses/BSD-3-Clause
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

#include <android/asset_manager_jni.h>
#include <android/bitmap.h>
#include <android/native_window_jni.h>
#include <android/native_window.h>

#include <android/log.h>

#include <jni.h>

#include <string>
#include <vector>

#include <platform.h>
#include <benchmark.h>

#include "rvm.h"

#include "ndkcamera.h"

#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>

#if __ARM_NEON
#include <arm_neon.h>
#endif // __ARM_NEON

static int draw_unsupported(cv::Mat& rgb)
{
    const char text[] = "unsupported";

    int baseLine = 0;
    cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 1.0, 1, &baseLine);

    int y = (rgb.rows - label_size.height) / 2;
    int x = (rgb.cols - label_size.width) / 2;

    cv::rectangle(rgb, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)),
                    cv::Scalar(255, 255, 255), -1);

    cv::putText(rgb, text, cv::Point(x, y + label_size.height),
                cv::FONT_HERSHEY_SIMPLEX, 1.0, cv::Scalar(0, 0, 0));

    return 0;
}

static int draw_fps(cv::Mat& rgb)
{
    // resolve moving average
    float avg_fps = 0.f;
    {
        static double t0 = 0.f;
        static float fps_history[10] = {0.f};

        double t1 = ncnn::get_current_time();
        if (t0 == 0.f)
        {
            t0 = t1;
            return 0;
        }

        float fps = 1000.f / (t1 - t0);
        t0 = t1;

        for (int i = 9; i >= 1; i--)
        {
            fps_history[i] = fps_history[i - 1];
        }
        fps_history[0] = fps;

        if (fps_history[9] == 0.f)
        {
            return 0;
        }

        for (int i = 0; i < 10; i++)
        {
            avg_fps += fps_history[i];
        }
        avg_fps /= 10.f;
    }

    char text[32];
    sprintf(text, "FPS=%.2f", avg_fps);

    int baseLine = 0;
    cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);

    int y = 0;
    int x = rgb.cols - label_size.width;

    cv::rectangle(rgb, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)),
                    cv::Scalar(255, 255, 255), -1);

    cv::putText(rgb, text, cv::Point(x, y + label_size.height),
                cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 0, 0));

    return 0;
}

static RVM* g_rvm = 0;
static ncnn::Mutex lock;
static InterFeatures g_feats;

class MyNdkCamera : public NdkCameraWindow
{
public:
    virtual void on_image_render(cv::Mat& rgb) const;
};

void MyNdkCamera::on_image_render(cv::Mat& rgb) const
{
    // rvm
    {
        ncnn::MutexLockGuard g(lock);

        if (g_rvm)
        {
            cv::Mat fgr;
            cv::Mat pha;
            cv::Mat seg;
            g_rvm->detect(rgb, g_feats, fgr, pha, seg);

            g_rvm->draw(rgb, fgr, pha, seg);
        }
        else
        {
            draw_unsupported(rgb);
        }
    }

    draw_fps(rgb);
}

static MyNdkCamera* g_camera = 0;

extern "C" {

JNIEXPORT jint JNI_OnLoad(JavaVM* vm, void* reserved)
{
    __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "JNI_OnLoad");

    g_camera = new MyNdkCamera;

    ncnn::create_gpu_instance();

    return JNI_VERSION_1_4;
}

JNIEXPORT void JNI_OnUnload(JavaVM* vm, void* reserved)
{
    __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "JNI_OnUnload");

    {
        ncnn::MutexLockGuard g(lock);

        delete g_rvm;
        g_rvm = 0;
    }

    ncnn::destroy_gpu_instance();

    delete g_camera;
    g_camera = 0;
}

// public native boolean loadModel(AssetManager mgr, int modelid, int sizeid, int intrainterid, int postprocid, int cpugpu);
JNIEXPORT jboolean JNICALL Java_org_example_project_RVMNcnn_loadModel(JNIEnv* env, jobject thiz, jobject assetManager, jint modelid, jint sizeid, jint intrainterid, jint postprocid, jint cpugpu)
{
    if (modelid < 0 || modelid > 1 || sizeid < 0 || sizeid > 6 || intrainterid < 0 || intrainterid > 1 || postprocid < 0 || postprocid > 2 || cpugpu < 0 || cpugpu > 2)
    {
        return JNI_FALSE;
    }

    AAssetManager* mgr = AAssetManager_fromJava(env, assetManager);

    __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "loadModel %p", mgr);

    const char* modeltypes[2] =
    {
        "mobilenetv3",
        "resnet50"
    };

    const int sizetypes[7] =
    {
        256,
        320,
        384,
        448,
        512,
        576,
        640
    };

    std::string parampath = std::string("rvm_") + modeltypes[(int)modelid] + ".ncnn.param";
    std::string modelpath = std::string("rvm_") + modeltypes[(int)modelid] + ".ncnn.bin";
    bool use_gpu = (int)cpugpu == 1;
    bool use_turnip = (int)cpugpu == 2;

    // reload
    {
        ncnn::MutexLockGuard g(lock);

        {
            // reset inter feats
            g_feats.r1.release();
            g_feats.r2.release();
            g_feats.r3.release();
            g_feats.r4.release();

            static int old_modelid = 0;
            static int old_cpugpu = 0;
            if (modelid != old_modelid || cpugpu != old_cpugpu)
            {
                // model or cpugpu changed
                delete g_rvm;
                g_rvm = 0;
            }
            old_modelid = modelid;
            old_cpugpu = cpugpu;

            ncnn::destroy_gpu_instance();

            if (use_turnip)
            {
                ncnn::create_gpu_instance("libvulkan_freedreno.so");
            }
            else if (use_gpu)
            {
                ncnn::create_gpu_instance();
            }

            if (!g_rvm)
            {
                g_rvm = new RVM;

                g_rvm->load(mgr, parampath.c_str(), modelpath.c_str(), use_gpu || use_turnip);
            }
            g_rvm->set_model_type((int)modelid);
            g_rvm->set_target_size(sizetypes[(int)sizeid]);
            g_rvm->set_intra_inter((int)intrainterid);

            // deep
            // fast
            // seg
            if (postprocid == 0)
                g_rvm->set_postproc_mode(false, true, false);
            if (postprocid == 1)
                g_rvm->set_postproc_mode(false, false, true);
            if (postprocid == 2)
                g_rvm->set_postproc_mode(true, false, false);
        }
    }

    return JNI_TRUE;
}

// public native boolean openCamera(int facing);
JNIEXPORT jboolean JNICALL Java_org_example_project_RVMNcnn_openCamera(JNIEnv* env, jobject thiz, jint facing)
{
    if (facing < 0 || facing > 1)
        return JNI_FALSE;

    __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "openCamera %d", facing);

    g_camera->open((int)facing);

    return JNI_TRUE;
}

// public native boolean closeCamera();
JNIEXPORT jboolean JNICALL Java_org_example_project_RVMNcnn_closeCamera(JNIEnv* env, jobject thiz)
{
    __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "closeCamera");

    g_camera->close();

    return JNI_TRUE;
}

// public native boolean setOutputWindow(Surface surface);
JNIEXPORT jboolean JNICALL Java_org_example_project_RVMNcnn_setOutputWindow(JNIEnv* env, jobject thiz, jobject surface)
{
    ANativeWindow* win = ANativeWindow_fromSurface(env, surface);

    __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "setOutputWindow %p", win);

    g_camera->set_window(win);

    return JNI_TRUE;
}

// public native boolean setBackgroundImage(Bitmap bitmap);
JNIEXPORT jboolean JNICALL Java_org_example_project_RVMNcnn_setBackgroundImage(JNIEnv* env, jobject thiz, jobject bitmap)
{
    if (!bitmap)
    {
        ncnn::MutexLockGuard g(lock);
        if (g_rvm)
        {
            g_rvm->clear_background_image();
        }
        return JNI_TRUE;
    }

    AndroidBitmapInfo info;
    if (AndroidBitmap_getInfo(env, bitmap, &info) < 0)
    {
        return JNI_FALSE;
    }

    if (info.format != ANDROID_BITMAP_FORMAT_RGBA_8888 && info.format != ANDROID_BITMAP_FORMAT_RGB_565)
    {
        return JNI_FALSE;
    }

    void* pixels;
    if (AndroidBitmap_lockPixels(env, bitmap, &pixels) < 0)
    {
        return JNI_FALSE;
    }

    cv::Mat mat;
    if (info.format == ANDROID_BITMAP_FORMAT_RGBA_8888)
    {
        mat = cv::Mat(info.height, info.width, CV_8UC4, pixels);
    }
    else if (info.format == ANDROID_BITMAP_FORMAT_RGB_565)
    {
        mat = cv::Mat(info.height, info.width, CV_8UC2, pixels);
        cv::cvtColor(mat, mat, cv::COLOR_BGR5652BGR);
    }

    cv::Mat background_bgr;
    if (mat.channels() == 4)
    {
        cv::cvtColor(mat, background_bgr, cv::COLOR_RGBA2BGR);
    }
    else
    {
        background_bgr = mat.clone();
    }

    AndroidBitmap_unlockPixels(env, bitmap);

    {
        ncnn::MutexLockGuard g(lock);
        if (g_rvm)
        {
            g_rvm->set_background_image(background_bgr);
        }
    }

    return JNI_TRUE;
}

}